/
graph.py
1402 lines (1241 loc) · 55.9 KB
/
graph.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import itertools
import logging
import operator
import os
import re
import sys
import time
from collections import defaultdict
from contextlib import contextmanager
from typing import Any, Callable, DefaultDict, Dict, List, Optional, Set, Tuple, Union
import sympy
import torch
import torch._logging
import torch.fx
from torch._decomp import get_decompositions
from torch._dynamo.utils import defake, dynamo_timed
from torch._logging import LazyString, trace_structured
from torch._subclasses.fake_tensor import FakeTensor
from torch.fx.experimental._backward_state import BackwardState
from torch.fx.experimental.sym_node import magic_methods, method_to_operator
from torch.fx.experimental.symbolic_shapes import (
free_unbacked_symbols,
has_free_symbols,
ShapeEnv,
SymTypes,
)
from torch.utils._mode_utils import no_dispatch
from . import config, ir
from .codegen.common import (
DeviceOpOverrides,
get_device_op_overrides,
get_scheduling_for_device,
get_wrapper_codegen_for_device,
register_backend_for_device,
)
from .codegen.cpp_wrapper_cpu import CppWrapperCpu
from .codegen.cpp_wrapper_cuda import CppWrapperCuda
from .codegen.wrapper import WrapperCodeGen
from .exc import (
CppWrapperCodeGenError,
LoweringException,
MissingOperatorWithDecomp,
MissingOperatorWithoutDecomp,
)
from .ir import (
Constant,
FixedLayout,
InputBuffer,
Pointwise,
Reduction,
StorageBox,
TensorBox,
)
from .lowering import (
constrain_to_fx_strides,
FALLBACK_ALLOW_LIST,
fallback_handler,
fallback_node_due_to_unsupported_type,
layout_constraints,
lowerings,
make_fallback,
needs_realized_inputs,
unsupported_output_tensor,
)
from .sizevars import SizeVarAllocator
from .utils import convert_shape_to_inductor, gather_origins, get_sympy_Expr_dtype
from .virtualized import V
log = logging.getLogger(__name__)
perf_hint_log = torch._logging.getArtifactLogger(__name__, "perf_hints")
output_code_log = torch._logging.getArtifactLogger(__name__, "output_code")
if config.is_fbcode():
from torch._inductor.fb.utils import log_module_code
else:
def log_module_code(*args, **kwargs):
pass
def supported_dtype_of_cpp_wrapper(dtype, cuda):
supported_dtype = {
torch.float32,
torch.float64,
torch.int64,
torch.int32,
torch.int16,
torch.int8,
torch.uint8,
torch.bool,
torch.bfloat16,
torch.complex32,
torch.complex64,
torch.complex128,
torch.float16,
}
if cuda:
supported_dtype.add(torch.float8_e4m3fn)
supported_dtype.add(torch.float8_e5m2)
supported_dtype.add(torch.float8_e4m3fnuz)
supported_dtype.add(torch.float8_e5m2fnuz)
return dtype in supported_dtype
def may_get_constant_buffer_dtype(constant_buffer):
assert isinstance(
constant_buffer, (sympy.Symbol, sympy.Expr, sympy.core.numbers.Integer)
), "get_constant_buffer_dtype only supports input of sympy.Symbol, sympy.Expr or sympy.core.numbers.Integer"
if isinstance(constant_buffer, sympy.core.numbers.Integer):
return torch.int64
if isinstance(constant_buffer, sympy.Expr):
return get_sympy_Expr_dtype(constant_buffer)
if constant_buffer.is_integer:
return torch.int64
elif constant_buffer.is_float:
return torch.float32
else:
return None
def is_magic_method(op):
magic_ops = {method_to_operator(m) for m in magic_methods}
return op in magic_ops
def getattr_recursive(obj, target):
target_atoms = target.split(".")
attr_itr = obj
for i, atom in enumerate(target_atoms):
if not hasattr(attr_itr, atom):
raise RuntimeError(
f"Node referenced nonexistent target {'.'.join(target_atoms[:i])}"
)
attr_itr = getattr(attr_itr, atom)
return attr_itr
class GraphLowering(torch.fx.Interpreter):
graph_outputs: List[ir.IRNode]
def symbolic_sizes_strides(self, ex: torch.Tensor):
"""
Support dynamic shapes and dynamic strides by assigning variables
to each dimension. We duck-shape tensors, so if two tensors
have the same size they get assigned the same symbolic variable.
"""
if self.reuse_shape_env:
return convert_shape_to_inductor(ex.size()), convert_shape_to_inductor(
ex.stride()
)
else:
from torch._dynamo.source import ConstantSource
# TODO: this should not be needed once #93059 lands
# https://github.com/pytorch/pytorch/pull/94031#discussion_r1096044816
# TODO: make a dedicated UnknownSource for this?
# NB: This is using the legacy default behavior from
# create_symbolic_sizes_strides_storage_offset but we hope we can
# just delete this entirely
source = ConstantSource(
f"__inductor_unknown_tensor_{len(self._shape_env.var_to_val)}"
)
(
size,
stride,
_,
) = self._shape_env.create_symbolic_sizes_strides_storage_offset(
ex,
source,
)
size = [i.node.expr if isinstance(i, torch.SymInt) else i for i in size]
stride = [i.node.expr if isinstance(i, torch.SymInt) else i for i in stride]
return size, stride
def static_sizes_strides(self, ex: torch.Tensor):
"""
Primarily used to weights
"""
size = [sympy.Integer(i) for i in ex.size()]
stride = [sympy.Integer(i) for i in ex.stride()]
return size, stride
def init_backend_registration(self):
if get_scheduling_for_device("cpu") is None:
from .codegen.cpp import CppScheduling
register_backend_for_device("cpu", CppScheduling, WrapperCodeGen)
if get_scheduling_for_device("cuda") is None:
from .codegen.cuda_combined_scheduling import CUDACombinedScheduling
# CUDACombinedScheduling combines Triton and CUDA C++ scheduling for CUDA devices via delegation
register_backend_for_device("cuda", CUDACombinedScheduling, WrapperCodeGen)
def __init__(
self,
gm: torch.fx.GraphModule,
example_inputs: Optional[List[torch.Tensor]] = None,
shape_env=None,
num_static_inputs=None,
graph_id=None,
cpp_wrapper=False,
aot_mode=False,
user_visible_outputs=frozenset(),
layout_opt=None,
extern_node_serializer=None,
is_inference=False,
is_const_graph=False,
const_output_index=None,
const_code=None,
const_module=None,
name=None,
):
super().__init__(gm)
self.example_inputs = example_inputs
self.layout_opt = (
layout_opt
if layout_opt is not None
else self.decide_layout_opt(gm, is_inference=is_inference)
)
self.num_channels_last_conv = 0
self.is_inference = is_inference
self.is_const_graph = is_const_graph
self.const_code = const_code
self.const_module = const_module
self.extra_traceback = False # we do our own error wrapping
if shape_env is None:
shape_env = ShapeEnv()
self.reuse_shape_env = False
else:
self._shape_env = shape_env
self.reuse_shape_env = True
self._shape_env = shape_env
self.sizevars = SizeVarAllocator(shape_env)
self.graph_input_names: List[str] = []
self.graph_inputs: Dict[str, TensorBox] = {}
self.graph_inputs_original: Dict[str, InputBuffer] = {}
self.device_types: Set[str] = (
const_module.device_types if const_module else set()
)
self.device_idxs: Set[int] = const_module.device_idxs if const_module else set()
self.cuda = False
self.buffers: List[ir.Buffer] = []
self.const_output_index: Dict[str, int] = (
const_output_index if const_output_index else {}
)
self.folded_constants: Set[str] = (
set(const_output_index.keys()) if const_output_index else set()
)
self.constants: Dict[str, torch.Tensor] = (
const_module.constants if const_module else {}
)
self.constant_reprs: Dict[str, str] = {}
self.removed_buffers: Set[str] = set()
self.removed_inplace_buffers: Set[str] = set()
self.mutated_buffers: Set[str] = set()
self.never_reuse_buffers: Set[str] = set()
self.inplaced_to_remove: Set[str] = set()
self.device_ops: DeviceOpOverrides = None # type: ignore[assignment]
self.wrapper_code: WrapperCodeGen = None # type: ignore[assignment]
# See `ProxyExecutor Design Note` in ir.py for more details
self.extern_kernel_nodes: List[ir.ExternKernelNode] = []
self.extern_node_serializer: Optional[
Callable[[List[ir.ExternKernelNode]], Any]
] = extern_node_serializer
self.current_node: torch.fx.Node = None # type: ignore[assignment]
self.num_static_inputs = num_static_inputs
self.lists: Dict[str, List[str]] = {}
self.mutated_inputs: Set[str] = set()
self.mutated_input_idxs: List[int] = []
self.name_to_buffer: Dict[str, ir.Buffer] = {}
self.name_to_users: DefaultDict[str, List[ir.IRNode]] = defaultdict(list)
self.creation_time = time.time()
self.name = name
self.cpp_wrapper = cpp_wrapper
# record multi_kernel choice for cpp_wrapper so the second pass knows
# which sub-kernel is picked. Copy cpp_wrapper to another variable
# since cpp_wrapper flag is set to false for the first pass of codegen.
self.record_multi_kernel_choice = cpp_wrapper
self.multi_kernel_to_choice: Dict[str, int] = {}
self.aot_mode = aot_mode
self.graph_id = graph_id
self.scheduler: "torch._inductor.scheduler.Scheduler" = None # type: ignore[assignment]
self.nodes_prefer_channels_last = (
self.find_nodes_prefer_channels_last() if self.layout_opt else set()
)
self._warned_fallback = {"aten.convolution_backward"}
self.user_visible_outputs = user_visible_outputs
self.cache_key: str = "" # This is the cache key for the compiled artifact
self.cache_path: str = "" # This is the path in the filesystem where the compiled artifact is stored
self.cache_linemap: List[
Tuple[int, str]
] = (
[]
) # This is the linemap used by the profiler to mark custom compiled kernels getting run
# Used if lowering encounters cases where cudagraphs are not supported
self.disable_cudagraphs_reason: Optional[str] = None
# only keeping one node per device for stack trace purposes
self.device_node_mapping: Dict[torch.device, torch.fx.Node] = {}
self.orig_gm: torch.fx.GraphModule = gm.__copy__()
self.dynamo_flat_name_to_original_fqn = self.module.meta.get(
"dynamo_flat_name_to_original_fqn", {}
)
self.allocated_constant_name = (
const_module.allocated_constant_name if const_module is not None else {}
)
self.init_backend_registration()
@staticmethod
def decide_layout_opt(gm, *, is_inference) -> bool:
"""
Decide if we should enable layout optimization for this graph based on
heuristics.
"""
if not config.layout_optimization:
return False
if config.force_layout_optimization:
return True
conv_nodes = [
n for n in gm.graph.nodes if n.target == torch.ops.aten.convolution.default
]
nconv = len(conv_nodes)
if nconv == 0:
return False
# For cpu backend and mkldnn enabled, we always use channels_last for better performance.
if (
torch.backends.mkldnn.enabled
and torch.backends.mkldnn.is_available()
and all(
n.args[idx].meta["val"].device == torch.device("cpu")
for n in conv_nodes
for idx in [0, 1]
)
):
return True
# Following models are skipped due to this:
# jx_nest_base
# volo_d1_224
if len(list(gm.graph.nodes)) >= 300 * nconv:
log.debug("Skipped layout opt because only a few conv")
return False
if any(
has_free_symbols(n.args[idx].meta["val"])
for n in conv_nodes
for idx in [0, 1]
):
log.debug(
"See perf regression with dynamic shape. Follow up in https://github.com/pytorch/pytorch/issues/102670"
)
return False
def is_grouped(n):
return n.args[-1] > 1 and n.args[1].meta["val"].size(1) > 1
def is_in_out_channel(n):
return (
n.args[1].meta["val"].size(0) * 2 <= n.args[1].meta["val"].size(1)
and n.args[1].meta["val"].size(2) > 1
)
def is_small_channel(n):
return (
n.args[1].meta["val"].size(0) <= 64
and n.args[1].meta["val"].size(1) <= 64
)
# only grouped convolutions benchmarked as slower in conv samples for inference only
if is_inference:
from torch.utils.flop_counter import FlopCounterMode
flop_counts: Dict[str, float] = defaultdict(float)
for node in conv_nodes:
success, args, kwargs = torch._inductor.fx_utils.get_fake_args_kwargs(
node
)
if success:
with FlopCounterMode(display=False) as flop_counter_mode:
with V.fake_mode:
node.target(*args, **kwargs)
counted_flops = flop_counter_mode.get_total_flops()
if is_grouped(node):
node_type = "grouped"
elif is_small_channel(node):
node_type = "small"
elif is_in_out_channel(node):
node_type = "in_out"
else:
node_type = "default"
flop_counts[node_type] += counted_flops
else:
log.debug("Conv inputs meta not found")
# average benchmarked channels last speedup / slowdown, < 1 is speedup.
# taken from the set of convolution inputs in benchmarks/dynamo/microbenchmarks/operator_inp_logs/torchbench_train/
# To regenerate these numbers follow https://gist.github.com/eellison/55d7a6ed6f39829d68ac56f95f4df5bb
GROUPED_MULTIPLIER = 1.358
DEFAULT_MULTIPLIER = 0.823
IN_OUT_MULTIPLIER = 0.725
SMALL_MULTIPLIER = 0.783
total_flops = sum(flop_counts.values())
# TODO - get different values per hardware
weighted_flops = (
flop_counts["grouped"] * GROUPED_MULTIPLIER
+ flop_counts["small"] * SMALL_MULTIPLIER
+ flop_counts["in_out"] * IN_OUT_MULTIPLIER
+ flop_counts["default"] * DEFAULT_MULTIPLIER
)
do_layout_opt = weighted_flops <= total_flops
if not do_layout_opt:
log.debug(
"Skipped layout opt in inference because weighted flops indicate slowdown, default: %d, channels last: %d",
total_flops,
weighted_flops,
)
return do_layout_opt
# Channels last layout can dramatically hurt grouped conv perf. E.g.
# Conv with arguments like
# {"input_shape": [32, 224, 112, 112], "weight_shape": [224, 112, 3, 3],
# "stride": [2, 2], "padding": [1, 1], "groups": 2}
# slows down 31x using channels last..
# But a lot of timm models use depthwise separable convolution which will
# result in grouped convolution with in-channel size == 1.
# For those grouped convolution, channels last still helps a lot.
# E.g.
# Conv with arguments
# {"input_shape": [128, 58, 56, 56], "weight_shape": [58, 1, 3, 3],
# "stride": [2, 2], "padding": [1, 1], "groups": 58}
# get 1.86x speedup with channels last layout.
#
# The following heuristics skip using channels-last if the model contains
# grouped convolution with in-channels > 1.
if any(map(is_grouped, conv_nodes)):
log.debug(
"Skip layout opt because found grouped convolution with >1 in_channels!"
)
return False
# For some models that contain convolution with larger in-channel than out-channel, applying
# channels last hurts performance.
# Following models are skipped due to this:
# - pytorch_unet
# - phlippe_densenet (slightly worse)
# - Background_Matting (1.22x -> 0.821x)
# - pytorch_CycleGAN_and_pix2pix (1.597x -> 1.294x)
if any(map(is_in_out_channel, conv_nodes)):
log.debug(
"Skip layout opt because some convolutions have smaller out_channel"
)
return False
# Following models are skipped due to this:
# - functorch_maml_omniglot
if all(map(is_small_channel, conv_nodes)):
log.debug("Skip layout opt because all convolution channels are too small")
return False
return True
def qualify_name(self, name: str) -> str:
"""Prepend the given name with the graph name if any."""
if self.name is not None:
return f"{self.name}_{name}"
return name
def make_subgraph(
self,
gm: torch.fx.GraphModule,
example_inputs: List[torch.Tensor],
subgraph_name: str,
) -> "GraphLowering":
"""
Make a subgraph of the current graph with all inherited
parts, except the graph module (`gm`) and `example_inputs`.
The subgraphs are lowered separately, but intended to be
inlined in the parent graph's codegening. Hence the need
for maintaining the same `shape_env` and other properties.
The subgraph name is qualified by the parent graph's name.
"""
return GraphLowering(
gm=gm,
example_inputs=example_inputs,
shape_env=self._shape_env,
cpp_wrapper=self.cpp_wrapper,
aot_mode=self.aot_mode,
extern_node_serializer=self.extern_node_serializer,
is_inference=self.is_inference,
name=self.qualify_name(subgraph_name),
)
def find_nodes_prefer_channels_last(self):
"""
The rule to decide if an node prefer channels last is simple.
1. if it's input/output of a convolution
2. if one of its user prefers channels last
We have rule 1 because cudnn runs a faster convolution kernel for channels last inputs;
Rule 2 is also important. It makes sure that indirect inputs to convolution also prefers
channels last.
Consider the scenario: conv -> batch-norm -> relu -> conv
Without rule 2, batch-norm output may use a contiguous layout. That will cause 2 extra copies:
1. the output of batch-norm should be channels last initially since its input is a conv's output.
Forcing the batch-norm's output to be contiguous results in the first copy
2. The second conv's input is initially contiguous. This layout is propagated from the batch-norm's output.
We need convert it to channels last layout which results in the second copy.
With rule 2, we makes sure all the tensors in the chain uses channels last layout. So both copies
can be saved.
"""
output_set = set()
for n in reversed(self.module.graph.nodes):
if n.target == torch.ops.aten.convolution.default:
output_set.add(n)
continue
for user in n.users:
if user in output_set:
output_set.add(n)
break
# need a second pass to add downstream nodes of those channel last nodes to the sets.
# This pass is especially needed to avoid mix-layout kernel inputs in backward pass.
#
# Let's say a conv-batchnorm 's output is passed to relu whose output is in turn returned
# from the fwd graph. Without this second pass, we will force relu's output to be contiguous.
# Then in the kernel in backward pass, the contiguous output of relu may be mix with other channels last
# tensors and passed to a kernel.
#
# This pass improve yolov3 training speedup from 1.116x (worse than disabling layout optimization speedup 1.196x) to 1.457x.
# It also improves dla102 training speedup from 1.240x (worse than disabling layout optimization speedup 1.523x) to 1.835x .
# This also helps the following models:
# - res2net101_26w_4s
# - res2net50_14w_8s
# - sebotnet33ts_256
for n in self.module.graph.nodes:
if n in output_set:
for child in n.users:
output_set.add(child)
return output_set
def warn_fallback(self, name):
if name not in self._warned_fallback:
self._warned_fallback.add(name)
perf_hint_log.info("Using FallbackKernel: %s", name)
def add_device_info(self, device: torch.device):
self.device_types.add(device.type)
if device.index is not None:
self.device_idxs.add(device.index)
if V.graph.current_node and device not in self.device_node_mapping:
self.device_node_mapping[device] = V.graph.current_node
@property
def fake_mode(self):
return V.fake_mode
def get_buffer(self, buffer_name: str):
if buffer_name in self.name_to_buffer:
return self.name_to_buffer[buffer_name]
if buffer_name in self.graph_inputs:
return self.graph_inputs[buffer_name]
return None
def get_dtype(self, buffer_name: str):
if buffer_name in self.constants:
return self.constants[buffer_name].dtype
if buffer_name in self.name_to_buffer:
return self.name_to_buffer[buffer_name].get_dtype()
if buffer_name in self.graph_inputs:
return self.graph_inputs[buffer_name].get_dtype()
m = re.match(r"(as_strided|reinterpret_tensor)\(([a-zA-Z0-9_]+),", buffer_name)
if m:
return self.get_dtype(m.group(1))
raise KeyError(f"could not find {buffer_name}")
def get_numel(self, buffer_name: str):
from .ir import MultiOutputLayout
if buffer_name in self.constants:
return self.constants[buffer_name].numel()
if buffer_name in self.name_to_buffer:
buf = self.name_to_buffer[buffer_name]
if isinstance(getattr(buf, "layout", None), MultiOutputLayout):
return 1
return buf.get_numel()
if buffer_name in self.graph_inputs:
return self.graph_inputs[buffer_name].get_numel()
raise KeyError(f"could not find {buffer_name}")
@dynamo_timed
def run(self, *args):
return super().run(*args)
def register_buffer(self, buffer: ir.Buffer):
name = self.qualify_name(f"buf{len(self.buffers)}")
self.buffers.append(buffer)
self.name_to_buffer[name] = buffer
# Skip empty CPU tensor so that CUDA graphs can succeed, see https://github.com/pytorch/pytorch/pull/114144
if not isinstance(buffer, ir.ComputedBuffer) or not buffer.is_zero_elements():
self.add_device_info(buffer.get_device())
return name
def register_list(self, buffer_names: List[str]):
name = self.qualify_name("list_" + "_".join(buffer_names))
self.lists[name] = buffer_names
return name
def register_users_of(self, node_output):
def register(value):
if isinstance(value, (list, tuple)):
for x in value:
register(x)
if isinstance(value, ir.IRNode):
if (
not hasattr(value, "data")
or not isinstance(value.data, ir.IRNode)
or not (
hasattr(value.data, "data")
and isinstance(value.data.data, ir.IRNode)
)
):
return
for read_name in value.get_read_names():
self.name_to_users[read_name].append(value)
register(node_output)
def mark_buffer_mutated(self, name: str):
"""
When a buffer is mutated we need to make sure all the reads to
the old version are realized before the mutation happens.
"""
assert isinstance(name, str)
self.mutated_buffers.add(name)
if name not in self.name_to_users:
return
for user in self.name_to_users[name]:
user.realize()
def add_tensor_constant(self, data, name=None):
def allocate(name):
if not config.aot_inductor.use_runtime_constant_folding:
for constant_name, value in self.constants.items():
if (
not data.is_mkldnn
and data.size() == value.size()
and data.stride() == value.stride()
and data.dtype == value.dtype
and data.device == value.device
and torch.eq(data, value).all()
):
return constant_name
if name is None:
name = f"constant{len(self.constants)}"
if name[0].isdigit():
name = f"constant_{name}"
name = self.qualify_name(name)
# We may generate a var name for each constant in the codegen.
# Let's only keep sane characters.
prefix = re.sub(r"[^a-zA-Z0-9_]", "_", name)
name = prefix
cnt = 0
while name in self.constants:
name = f"{prefix}_{cnt}"
cnt += 1
self.constants[name] = data
self.constant_reprs[name] = (
f"{data.device!r} {data.dtype!r} "
f"{tuple(data.size())!r} {tuple(data.stride())!r} "
f"{hash(data):x}"
)
return name
new_name = allocate(name)
self.allocated_constant_name[new_name] = name
return TensorBox.create(
ir.ConstantBuffer(
new_name,
FixedLayout(data.device, data.dtype, *self.static_sizes_strides(data)),
)
)
def constant_name(self, name: str, device_override: Optional[torch.device]):
"""
We AOT copy constants to the devices they are needed on.
If device_override doesn't match the constant's device, then
copy it and return a different name.
"""
if self.constants[name].device == device_override or device_override is None:
return name
alt_name = f"{name}_{device_override.type}{device_override.index or 0}"
if alt_name not in self.constants:
self.constants[alt_name] = self.constants[name].to(device_override)
return alt_name
def placeholder(self, target: str, args, kwargs):
example = super().placeholder(target, args, kwargs)
self.graph_input_names.append(target)
if isinstance(example, SymTypes):
expr = example.node.expr
self.graph_inputs[target] = expr
return expr
elif isinstance(example, (int, bool, float)):
expr = sympy.sympify(example)
self.graph_inputs[target] = expr
return expr
if isinstance(example, BackwardState):
# Ignored arg, must be unused
# Alternately we could filter this out in AotAutograd
return None
assert isinstance(example, torch.Tensor), example
# todo(chilli): We can remove the last check once we turn buffers into
# static shape tensors. That's a hack to workaround Inductor believing
# the buffer should be static but us passing in a fake tensor with
# symbolic shapes.
if not example._has_symbolic_sizes_strides:
# the first N inputs are weights
sizes, strides = self.static_sizes_strides(example)
else:
sizes, strides = self.symbolic_sizes_strides(example)
# TODO(jansel): handle input aliasing
target = self.qualify_name(target)
tensor = TensorBox.create(
InputBuffer(
target,
FixedLayout(example.device, example.dtype, sizes, strides),
)
)
self.graph_inputs[target] = tensor
self.graph_inputs_original[target] = tensor.data.data
self.add_device_info(example.device)
return tensor
def call_function(self, target, args, kwargs):
if target is operator.getitem and isinstance(args[0], (list, tuple, dict)):
return super().call_function(target, args, kwargs)
if hasattr(target, "_inductor_lowering_function"):
# passthrough lowerings from .pattern_matcher
return target(*args, **kwargs)
def get_custom_op_layout_constraints(target, args, kwargs):
# Custom operations that require preserving stride order
# which run through implicit fallback must constrain their
# arguments' fx strides
layout_constraint = None
if torch._C.Tag.needs_fixed_stride_order in target.tags:
# We have to set the current args because call_function will immediately
# evaluate this lowering after creating the fallback, without evaluating
# the layout constraint
args, kwargs = constrain_to_fx_strides(
self.current_node, *args, **kwargs
)
# Also register the layout constraint so when the fallback
# is used again, we can constrain the args to the same layout
layout_constraint = constrain_to_fx_strides
return layout_constraint, args, kwargs
if target not in lowerings:
assert isinstance(
target, torch._ops.OpOverload
), f"{target} is not an OpOverload"
base_name = target.name().split(".")[0]
if base_name in FALLBACK_ALLOW_LIST:
make_fallback(target)
elif config.implicit_fallbacks:
layout_constraint, args, kwargs = get_custom_op_layout_constraints(
target, args, kwargs
)
error = (
MissingOperatorWithDecomp
if get_decompositions([target])
else MissingOperatorWithoutDecomp
)
log.info(
"Creating implicit fallback for:\n%s",
error.operator_str(target, args, kwargs),
)
make_fallback(target, layout_constraint)
elif get_decompositions([target]):
# There isn't a good way to dynamically patch this in
# since AOT Autograd already ran. The error message tells
# the user how to fix it.
raise MissingOperatorWithDecomp(target, args, kwargs)
else:
raise MissingOperatorWithoutDecomp(target, args, kwargs)
try:
log.debug(" via %s", lowerings[target])
out = lowerings[target](*args, **kwargs)
return out
except Exception as e:
raise LoweringException(e, target, args, kwargs).with_traceback(
e.__traceback__
) from None
@staticmethod
def can_inline_constant(t: torch.Tensor) -> bool:
"""
True if this is a small constant attr that will be inlined.
"""
return len(t.shape) == 1 and t.shape[0] <= 8
def get_attr(self, target, args, kwargs):
# this is a constant
value = getattr_recursive(self.module, target)
if isinstance(value, torch.fx.GraphModule):
return ir.Subgraph(name=target, graph_module=value)
if (
config.aot_inductor.use_runtime_constant_folding
or config.always_keep_tensor_constants
or unsupported_output_tensor(value)
):
return self.add_tensor_constant(value, target)
with no_dispatch():
if value.shape == ():
return Constant(value.item(), value.dtype, value.device)
if self.can_inline_constant(value):
# tensor lowering has constant inlining logic
from .lowering import tensor
return tensor(value.tolist(), dtype=value.dtype, device=value.device)
return self.add_tensor_constant(value, target)
def call_module(self, target, args, kwargs):
raise AssertionError()
def call_method(self, target, args, kwargs):
raise AssertionError()
def output(self, target, args, kwargs):
result = super().output(target, args, kwargs)
if not isinstance(result, (tuple, list)):
# nested subgraphs can have singleton outputs
result = (result,)
assert isinstance(result, (tuple, list)), type(result)
assert all(
isinstance(
x,
(
TensorBox,
ir.Constant,
type(None),
ir.ConstantBuffer,
sympy.Expr,
sympy.logic.boolalg.Boolean,
int,
),
)
for x in result
), result
fx_node_args = V.graph.current_node.args[0] # type: ignore[arg-type]
if not isinstance(fx_node_args, (tuple, list)):
# nested subgraphs can have singleton outputs
fx_node_args = (fx_node_args,)
result = [ir.ExternKernel.realize_input(x) for x in result]
result_correct_strides = []
assert len(fx_node_args) == len(result)
for r, fx_node in zip(result, fx_node_args):
if not isinstance(r, (ir.TensorBox, ir.BaseView)):
result_correct_strides.append(r)
else:
# AOT Autograd tries to detect stride divergence of inductor from output metadata.
# Here, we try to avoid spurious divergence by matching insignificant strides such as
result_correct_strides.append(
self.match_insignificant_strides(r, fx_node.meta["val"].stride())
)
self.graph_outputs = result_correct_strides
value: ir.IRNode
for name, value in self.graph_inputs.items():
assert isinstance(
value, (TensorBox, sympy.Expr)
), f"Unsupported inductor graph input type: {type(value)}"
if not isinstance(value, TensorBox):
continue
value.realize()
assert isinstance(value, TensorBox)
value = value.data
assert isinstance(value, ir.StorageBox)
value_storage_box = value
value = value.data
if not isinstance(value, InputBuffer) or value.get_name() != name:
# one of our inputs was mutated, need to turn that into a copy
ir.MutationLayoutSHOULDREMOVE.realize_into(
value, self.graph_inputs_original[name]
)
# replace output with mutated input
try:
ind = self.graph_outputs.index(value_storage_box)
self.graph_outputs[ind] = self.graph_inputs_original[name]
except ValueError:
pass
self.finalize()
log.debug(
"Force channels last inputs for %d conv for the current graph with id %d",
self.num_channels_last_conv,
self.graph_id if self.graph_id is not None else -1,
)
def finalize(self):
for buf in self.buffers:
buf.decide_layout()
@contextmanager
def set_current_node(self, node: torch.fx.Node):
old = self.current_node
try:
self.current_node = node
yield
finally:
self.current_node = old
def match_insignificant_strides(
self,
tensor,
meta_strides_inp: Tuple[Union[int, torch.SymInt], ...],
) -> ir.TensorBox:
# should have already been realized
assert torch._inductor.ir.is_storage_and_layout(tensor)
meta_strides = [
s.node.expr if isinstance(s, torch.SymInt) else s for s in meta_strides_inp
]
if all(
self.sizevars.statically_known_equals(s1, s2)
for s1, s2 in zip(meta_strides, tensor.get_stride())
):
return tensor
def significant_strides_equal(shape, meta_strides, tensor_strides):
for dim, s1, s2 in zip(shape, meta_strides, tensor_strides):
if self.sizevars.statically_known_leq(dim, 1): # type: ignore[arg-type]
continue
if not self.sizevars.statically_known_equals(s1, s2):
return False
return True
if not significant_strides_equal(
tensor.get_size(), meta_strides, tensor.get_stride()
):
return tensor
storage, old_layout = torch._inductor.ir.as_storage_and_layout(tensor)
new_stride = list(old_layout.stride)
for i, s in enumerate(tensor.get_size()):
if self.sizevars.statically_known_leq(s, 1): # type: ignore[arg-type]
new_stride[i] = meta_strides[i]
new_layout = torch._inductor.ir.FixedLayout(
old_layout.device,
old_layout.dtype,
old_layout.size,
new_stride,
old_layout.offset,
)
return ir.TensorBox(torch._inductor.ir.ReinterpretView(storage, new_layout))
def run_node(self, n: torch.fx.Node):