forked from pytorch/pytorch
-
Notifications
You must be signed in to change notification settings - Fork 0
/
utils.py
1495 lines (1287 loc) · 69.6 KB
/
utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
r"""
The torch.onnx module contains functions to export models into the ONNX
IR format. These models can be loaded with the ONNX library and then
converted to models which run on other deep learning frameworks.
"""
import torch
import torch.jit
import torch.autograd
import torch.serialization
import re
import collections
import contextlib
import copy
import numbers
import warnings
from torch._six import string_classes
from torch.jit import _unique_state_dict
from torch.onnx import ONNX_ARCHIVE_MODEL_PROTO_NAME, ExportTypes, OperatorExportTypes, TrainingMode, CheckerError
from torch._C import ListType, OptionalType, _propagate_and_assign_input_shapes, _check_onnx_proto
from typing import List, Tuple, Union
# the flag to tell the user whether it's in the middle of ONNX export or not
__IN_ONNX_EXPORT = False
def is_in_onnx_export():
global __IN_ONNX_EXPORT
return __IN_ONNX_EXPORT
# Skip check due to cannot import IValue from torch._C
_params_dict = {} # type: ignore[var-annotated]
@contextlib.contextmanager
def select_model_mode_for_export(model, mode):
if not isinstance(model, torch.jit.ScriptFunction):
is_originally_training = model.training
if mode is None:
mode = TrainingMode.EVAL
# if the model is in training mode but the user did not specify
# to export the model in training mode, export the model in inference
# mode (default) and warn them
if is_originally_training:
warnings.warn("You are exporting the model to ONNX while in training mode with "
"'train' parameter not specified. The model will default to inference mode export. "
"If you wish to export a training amenable ONNX model, specify training=TrainingMode.TRAINING or "
"training=TrainingMode.PRESERVE (to preserve the original model state) in torch.onnx.export().")
# if mode == TrainingMode.EVAL or (mode == TrainingMode.PRESERVE and not is_originally_training) => is_training = False
is_export_training = False
# ONNX opset 12 has better support for training amenable models, with updated
# versions of the dropout and batch_norm operators
if mode == TrainingMode.TRAINING or (mode == TrainingMode.PRESERVE and is_originally_training):
from torch.onnx.symbolic_helper import _export_onnx_opset_version
if _export_onnx_opset_version < 12:
warnings.warn("You are exporting the model in training mode with onnx opset version {}. "
"Opset versions lower than opset 12 will not be able to export nodes such as "
"Dropout and BatchNorm correctly.".format(_export_onnx_opset_version))
is_export_training = True
from torch.onnx.symbolic_helper import _set_training_mode
_set_training_mode(is_export_training)
model.train(is_export_training)
try:
yield
finally:
if not isinstance(model, torch.jit.ScriptFunction):
model.train(is_originally_training)
@contextlib.contextmanager
def disable_apex_o2_state_dict_hook(model):
# Apex O2 hook state_dict to return fp16 weights as fp32.
# Exporter cannot identify them as same tensors.
# Since this hook is only used by optimizer, it is safe to
# remove this hook while exporting.
if not isinstance(model, torch.jit.ScriptFunction):
tmp_map = {} # type: ignore[var-annotated]
for module in model.modules():
for k, v in module._state_dict_hooks.items():
if type(v).__name__ == 'O2StateDictHook':
if module not in tmp_map:
tmp_map[module] = {}
tmp_map[module][k] = v
if module in tmp_map:
for k in tmp_map[module].keys():
module._state_dict_hooks.pop(k)
try:
yield
finally:
if not isinstance(model, torch.jit.ScriptFunction):
for module, m_map in tmp_map.items():
for k, v in m_map.items():
module._state_dict_hooks[k] = v
@contextlib.contextmanager
def exporter_context(model, mode):
with select_model_mode_for_export(model, mode) as mode_ctx, \
disable_apex_o2_state_dict_hook(model) as apex_ctx:
yield (mode_ctx, apex_ctx)
def export(model, args, f, export_params=True, verbose=False, training=None,
input_names=None, output_names=None, operator_export_type=None,
opset_version=None, do_constant_folding=True, dynamic_axes=None,
keep_initializers_as_inputs=None, custom_opsets=None,
export_modules_as_functions=False):
if operator_export_type is None:
if torch.onnx.PYTORCH_ONNX_CAFFE2_BUNDLE:
operator_export_type = OperatorExportTypes.ONNX_ATEN_FALLBACK
else:
operator_export_type = OperatorExportTypes.ONNX
_export(model, args, f, export_params, verbose, training, input_names, output_names,
operator_export_type=operator_export_type, opset_version=opset_version,
do_constant_folding=do_constant_folding, dynamic_axes=dynamic_axes,
keep_initializers_as_inputs=keep_initializers_as_inputs,
custom_opsets=custom_opsets, export_modules_as_functions=export_modules_as_functions)
def _is_constant_tensor_list(node):
if node.kind() != "prim::Constant":
return False
output_type = node.output().type()
if output_type.isSubtypeOf(ListType.ofTensors()):
return True
if output_type.isSubtypeOf(ListType(OptionalType.ofTensor())):
return True
# ONNX can't handle constants that are lists of tensors, which can
# get generated in constant prop. So we split them back into prim::ListConstructs
def _split_tensor_list_constants(g, block):
for node in block.nodes():
for subblock in node.blocks():
_split_tensor_list_constants(g, subblock)
if _is_constant_tensor_list(node):
inputs = []
for val in node.output().toIValue():
input = g.insertConstant(val)
input.node().moveBefore(node)
input.node().copyMetadata(node)
inputs.append(input)
lc = (g.create("prim::ListConstruct", inputs)
.insertBefore(node)
.output()
.setType(ListType.ofTensors()))
lc.node().copyMetadata(node)
node.output().replaceAllUsesWith(lc)
def _export_graph_to_file(graph, operator_export_type):
print('[pth] start _optimize_graph')
graph = torch._C._jit_pass_onnx(graph, operator_export_type)
import random, string
seed = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(4))
proto, export_map, val_use_external_data_format = graph._export_onnx({}, 12, {}, False, operator_export_type, False, False, {}, True, f'/bert_ort/wechi/model_{seed}', {})
print(graph)
assert(len(export_map) == 0)
with torch.serialization._open_file_like(f'/bert_ort/wechi/model_{seed}.onnx', "wb") as opened_file:
opened_file.write(proto)
print('[pth] done _optimize_graph')
import onnx
m = onnx.load(f'/bert_ort/wechi/model_{seed}.onnx')
print(m)
return graph
def _optimize_graph_1(graph, operator_export_type, _disable_torch_constant_prop=False, fixed_batch_size=False,
params_dict=None, dynamic_axes=None, input_names=None, module=None):
print('[pth] start _optimize_graph')
#print(graph)
#for i in graph.inputs():
# print(i)
#for o in graph.outputs():
# print(o)
# Inline everything
#[4:38 PM] Bowen Bao
from torch.onnx.symbolic_helper import _set_onnx_shape_inference
_set_onnx_shape_inference(True)
torch._C._jit_pass_inline(graph)
# Remove fork/wait nodes
torch._C._jit_pass_inline_fork_wait(graph)
torch._C._jit_pass_lint(graph)
torch._C._jit_pass_lower_all_tuples(graph)
# we now record some ops like ones/zeros
# into a trace where we previously recorded constants.
# use constant prop to maintain our current level of onnx support
# without implementing symbolics for all of them
if _disable_torch_constant_prop is False:
torch._C._jit_pass_constant_propagation(graph)
_split_tensor_list_constants(graph, graph)
# run dce to eliminate dead parts of the graph that might have been
# left behind by things like symbolic_override
torch._C._jit_pass_dce(graph)
torch._C._jit_pass_lint(graph)
torch._C._jit_pass_canonicalize_graph_fuser_ops(graph)
torch._C._jit_pass_lint(graph)
torch._C._jit_pass_peephole(graph, True)
torch._C._jit_pass_fuse_addmm(graph)
torch._C._jit_pass_lint(graph)
from torch.onnx.symbolic_helper import _onnx_shape_inference, _set_opset_version, _set_operator_export_type
_set_opset_version(12)
_set_operator_export_type(OperatorExportTypes.ONNX_ATEN_FALLBACK)
from torch.onnx.symbolic_helper import _export_onnx_opset_version
torch._C._jit_pass_peephole(graph, True)
torch._C._jit_pass_lower_all_tuples(graph)
# in _jit_pass_onnx, symbolic functions are called for each node for conversion.
# However, there are nodes that cannot be converted without additional context.
# For example, the number of outputs from split (and whether it is static or dynamic) is unknown
# until the point where it is unpacked by listUnpack node.
# This pass does a preprocess, and prepares the nodes such that enough context can be received
# by the symbolic function.
torch._C._jit_pass_onnx_remove_inplace_ops_for_onnx(graph, module)
torch._C._jit_pass_onnx_preprocess(graph)
# onnx does not support tuples, so try to remove them
torch._C._jit_pass_lint(graph)
# onnx only supports tensors, but 1 / 2 = 0.5 and tensor(1) / tensor(2) = 0
torch._C._jit_pass_prepare_division_for_onnx(graph)
torch._C._jit_pass_onnx_remove_print(graph)
torch._C._jit_pass_onnx_preprocess_caffe2(graph)
if operator_export_type == OperatorExportTypes.ONNX_ATEN_FALLBACK:
torch.onnx.symbolic_helper._quantized_ops.clear()
# Unpack quantized weights for conv and linear ops and insert into graph.
torch._C._jit_pass_onnx_unpack_quantized_weights(graph, {})
# Insert permutes before and after each conv op to ensure correct order.
torch._C._jit_pass_onnx_quantization_insert_permutes(graph, {})
# Find consecutive permutes that are no-ops and remove them.
torch._C._jit_pass_custom_pattern_based_rewrite_graph("""
graph(%Pi):
%Pq = quantized::nhwc2nchw(%Pi)
%Pr = quantized::nchw2nhwc(%Pq)
return (%Pr)""", """
graph(%Ri):
return (%Ri)""", graph)
# onnx only supports tensors, so we turn all out number types into tensors
torch._C._jit_pass_erase_number_types(graph)
if _onnx_shape_inference:
input_names = [] if input_names is None else input_names
dynamic_axes = {} if dynamic_axes is None else dynamic_axes
torch._C._jit_pass_onnx_set_dynamic_input_shape(graph, dynamic_axes, input_names)
torch._C._jit_pass_onnx_lint(graph)
graph = torch._C._jit_pass_onnx(graph, operator_export_type)
torch._C._jit_pass_onnx_lint(graph)
torch._C._jit_pass_lint(graph)
print('[utils.py] _jit_pass_onnx_scalar_type_analysis\n', graph)
torch._C._jit_pass_onnx_scalar_type_analysis(graph, True, _export_onnx_opset_version)
print('[utils.py] _jit_pass_onnx_scalar_type_analysis done\n', graph)
torch._C._jit_pass_lint(graph)
torch._C._jit_pass_onnx_peephole(graph, _export_onnx_opset_version, fixed_batch_size)
torch._C._jit_pass_lint(graph)
# graph is not a valid jit graph anymore because types have been replaced
# (e.g. int with Tensor), so it now contains operators that don't actually
# exist. We can't run normal dead code elimination because it'd fail trying
# to look up if an operator has side effects, but we can run a dead code
# elimination variant that doesn't need to look up if an op has side effects.
torch._C._jit_pass_dce_allow_deleting_nodes_with_side_effects(graph)
torch._C._jit_pass_lint(graph)
graph = torch._C._jit_pass_canonicalize(graph)
torch._C._jit_pass_lint(graph)
if _onnx_shape_inference:
torch._C._jit_pass_onnx_graph_shape_type_inference(graph, {}, _export_onnx_opset_version)
import random, string
seed = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(4))
proto, export_map, val_use_external_data_format = graph._export_onnx({}, 12, {}, False, operator_export_type, False, False, {}, True, f'/bert_ort/wechi/model_{seed}', {})
assert(len(export_map) == 0)
model_location = f'/bert_ort/wechi/model_{seed}.onnx'
with torch.serialization._open_file_like(model_location, "wb") as opened_file:
opened_file.write(proto)
import onnx
print('Compare final JIT graph and ONNX model in exporter: ', model_location)
print('JIT model:')
print(graph)
m_ = onnx.load(model_location)
print('ONNX model:')
print(m_)
print('[pth] done _optimize_graph')
return model_location
def _optimize_graph(graph, operator_export_type, _disable_torch_constant_prop=False, fixed_batch_size=False,
params_dict=None, dynamic_axes=None, input_names=None, module=None):
print('[pth] start _optimize_graph')
print(graph)
for i in graph.inputs():
print(i)
for o in graph.outputs():
print(o)
# Inline everything
torch._C._jit_pass_inline(graph)
# Remove fork/wait nodes
torch._C._jit_pass_inline_fork_wait(graph)
torch._C._jit_pass_lint(graph)
torch._C._jit_pass_lower_all_tuples(graph)
# we now record some ops like ones/zeros
# into a trace where we previously recorded constants.
# use constant prop to maintain our current level of onnx support
# without implementing symbolics for all of them
if _disable_torch_constant_prop is False:
torch._C._jit_pass_constant_propagation(graph)
_split_tensor_list_constants(graph, graph)
# run dce to eliminate dead parts of the graph that might have been
# left behind by things like symbolic_override
torch._C._jit_pass_dce(graph)
torch._C._jit_pass_lint(graph)
torch._C._jit_pass_canonicalize_graph_fuser_ops(graph)
torch._C._jit_pass_lint(graph)
torch._C._jit_pass_peephole(graph, True)
torch._C._jit_pass_fuse_addmm(graph)
torch._C._jit_pass_lint(graph)
from torch.onnx.symbolic_helper import _onnx_shape_inference, _set_opset_version, _set_operator_export_type
_set_opset_version(12)
_set_operator_export_type(OperatorExportTypes.ONNX_ATEN_FALLBACK)
from torch.onnx.symbolic_helper import _export_onnx_opset_version
torch._C._jit_pass_peephole(graph, True)
torch._C._jit_pass_lower_all_tuples(graph)
# in _jit_pass_onnx, symbolic functions are called for each node for conversion.
# However, there are nodes that cannot be converted without additional context.
# For example, the number of outputs from split (and whether it is static or dynamic) is unknown
# until the point where it is unpacked by listUnpack node.
# This pass does a preprocess, and prepares the nodes such that enough context can be received
# by the symbolic function.
torch._C._jit_pass_onnx_remove_inplace_ops_for_onnx(graph, module)
torch._C._jit_pass_onnx_preprocess(graph)
# onnx does not support tuples, so try to remove them
torch._C._jit_pass_lint(graph)
# onnx only supports tensors, but 1 / 2 = 0.5 and tensor(1) / tensor(2) = 0
torch._C._jit_pass_prepare_division_for_onnx(graph)
torch._C._jit_pass_onnx_remove_print(graph)
torch._C._jit_pass_onnx_preprocess_caffe2(graph)
if operator_export_type == OperatorExportTypes.ONNX_ATEN_FALLBACK:
torch.onnx.symbolic_helper._quantized_ops.clear()
# Unpack quantized weights for conv and linear ops and insert into graph.
torch._C._jit_pass_onnx_unpack_quantized_weights(graph, {})
# Insert permutes before and after each conv op to ensure correct order.
torch._C._jit_pass_onnx_quantization_insert_permutes(graph, {})
# Find consecutive permutes that are no-ops and remove them.
torch._C._jit_pass_custom_pattern_based_rewrite_graph("""
graph(%Pi):
%Pq = quantized::nhwc2nchw(%Pi)
%Pr = quantized::nchw2nhwc(%Pq)
return (%Pr)""", """
graph(%Ri):
return (%Ri)""", graph)
# onnx only supports tensors, so we turn all out number types into tensors
torch._C._jit_pass_erase_number_types(graph)
if _onnx_shape_inference:
input_names = [] if input_names is None else input_names
dynamic_axes = {} if dynamic_axes is None else dynamic_axes
torch._C._jit_pass_onnx_set_dynamic_input_shape(graph, dynamic_axes, input_names)
torch._C._jit_pass_onnx_lint(graph)
onnx_graph = torch._C._jit_pass_onnx(graph, operator_export_type)
torch._C._jit_pass_onnx_lint(onnx_graph)
torch._C._jit_pass_lint(onnx_graph)
torch._C._jit_pass_onnx_scalar_type_analysis(onnx_graph, True, _export_onnx_opset_version)
torch._C._jit_pass_lint(onnx_graph)
torch._C._jit_pass_onnx_peephole(onnx_graph, _export_onnx_opset_version, fixed_batch_size)
torch._C._jit_pass_lint(onnx_graph)
# graph is not a valid jit graph anymore because types have been replaced
# (e.g. int with Tensor), so it now contains operators that don't actually
# exist. We can't run normal dead code elimination because it'd fail trying
# to look up if an operator has side effects, but we can run a dead code
# elimination variant that doesn't need to look up if an op has side effects.
torch._C._jit_pass_dce_allow_deleting_nodes_with_side_effects(onnx_graph)
torch._C._jit_pass_lint(onnx_graph)
graph = torch._C._jit_pass_canonicalize(onnx_graph)
torch._C._jit_pass_lint(onnx_graph)
if _onnx_shape_inference:
torch._C._jit_pass_onnx_graph_shape_type_inference(onnx_graph, params_dict, _export_onnx_opset_version)
import random, string
seed = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(4))
proto, export_map, val_use_external_data_format = onnx_graph._export_onnx({}, 12, {}, False, operator_export_type, False, False, {}, True, f'/bert_ort/wechi/model_{seed}', {})
print(proto)
assert(len(export_map) == 0)
with torch.serialization._open_file_like(f'/bert_ort/wechi/model_{seed}.onnx', "wb") as opened_file:
opened_file.write(proto)
print('[pth] done _optimize_graph')
return onnx_graph
# We accept dictionaries and strings as ONNX inputs,
# but they should be only for configuration use.
# we detect here if these inputs are modified, and if so
# we warn the user that the changes won't take effect in the
# traced ONNX graph
def warn_on_static_input_change(input_states):
for input, traced_input in zip(input_states[0], input_states[1]):
if isinstance(input, dict):
if list(input.keys()) != list(traced_input.keys()):
warning = "We detected that you are modifying a dictionary that is an input to your " \
"model. " \
"Note that dictionaries are allowed as inputs in ONNX but they should be " \
"handled with care. " \
"Usages of dictionaries is not recommended, and should not be used except " \
"for configuration use. " \
"Also note that the order and values of the keys must remain the same. "
warnings.warn(warning)
elif isinstance(input, str):
if input != traced_input:
warning = "The model seems to have string inputs/outputs. " \
"Note that strings will not appear as inputs/outputs of the ONNX graph. "
warnings.warn(warning)
def _resolve_args_by_export_type(arg_name, arg_value, operator_export_type):
# This helper method resolves the arguments that are ignored when export_type != operator_export_type.ONNX
if operator_export_type is not operator_export_type.ONNX:
if arg_value is True:
warnings.warn("`{}' can be set to True only when 'operator_export_type' is "
"`ONNX`. Since 'operator_export_type' is not set to 'ONNX', "
"`{}` argument will be ignored.".format(arg_name, arg_name))
arg_value = False
return arg_value
def _decide_keep_init_as_input(keep_initializers_as_inputs, operator_export_type,
opset_version):
# This method encapsulates the logic to decide whether the initializers in the graph
# should be listed as ONNX graph inputs (i.e., whether to choose ONNX IR v3 or v4).
# If keep_initializers_as_inputs is not specified (None), then we decide whether to keep
# initializers as graph inputs (val_keep_init_as_ip) based on export type. If export type
# is ONNX, then do not keep initializers as input (val_keep_init_as_ip=False). For all other
# export types keep initializers as input (val_keep_init_as_ip=True).
# If keep_initializers_as_inputs is specified, then respect it. Unless opset version <= 8,
# in which case it must be ignored because for opset version <= 8, all initializers MUST be
# part of graph input (only ONNX IR v3 is allowed), i.e. val_keep_init_as_ip=True.
# Special handling is needed for opset version 8 or lower, because irrespective
# of user input for keep_initializers_as_inputs, the graph must follow ONNX IR v3
# semantics, i.e. all initializers must be listed as ONNX graph input.
if opset_version < 9:
if keep_initializers_as_inputs is False:
warnings.warn("Setting 'keep_initializers_as_inputs=False' for opset version"
"8 or lower would lead to an invalid ONNX graph. Therefore, "
"'keep_initializers_as_inputs=False' is ignored during export."
"Exported model will have initializers as graph inputs (compliant "
" to ONNX IR v3).")
return True # i.e. True == initializers are part of graph input (ONNX IR v3)
val_keep_init_as_ip = True if keep_initializers_as_inputs is None else keep_initializers_as_inputs
if keep_initializers_as_inputs is None and operator_export_type is OperatorExportTypes.ONNX:
val_keep_init_as_ip = False
return val_keep_init_as_ip
def _decide_add_node_names(add_node_names, operator_export_type):
return _resolve_args_by_export_type("add_node_names", add_node_names, operator_export_type)
def _decide_constant_folding(do_constant_folding, operator_export_type, training):
do_constant_folding = _resolve_args_by_export_type("do_constant_folding", do_constant_folding, operator_export_type)
if do_constant_folding and (training is not None and training is not TrainingMode.EVAL):
warnings.warn("It is recommended that constant folding be turned off ('do_constant_folding=False') "
"when exporting the model in training-amenable mode, i.e. with 'training=TrainingMode.TRAIN' "
"or 'training=TrainingMode.PRESERVE' (when model is in training mode). Otherwise, some "
"learnable model parameters may not translate correctly in the exported ONNX model "
"because constant folding mutates model parameters. Please consider "
"turning off constant folding or setting the training=TrainingMode.EVAL.")
return do_constant_folding
def _decide_input_format(model, args):
import inspect
try:
sig = inspect.signature(model.forward)
ordered_list_keys = list(sig.parameters.keys())
if isinstance(args[-1], dict):
args_dict = args[-1]
args = list(args)[:-1]
n_nonkeyword = len(args)
for optional_arg in ordered_list_keys[n_nonkeyword:]:
if optional_arg in args_dict:
args.append(args_dict[optional_arg])
# Check if this arg has a default value
else:
param = sig.parameters[optional_arg]
if param.default is param.empty:
args.append(None)
else:
args.append(param.default)
args = tuple(args)
return args
# Cases of models without forward functions and dict inputs
except (AttributeError, ValueError):
warnings.warn("Model has no forward function")
return args
# Cases of models with no input args
except IndexError:
warnings.warn("No input args")
return args
except Exception as e:
warnings.warn("Skipping _decide_input_format\n {}".format(e.args[0]))
return args
def _trace(func, args, operator_export_type, return_outs=False):
# Special case for common case of passing a single Tensor
if isinstance(args, torch.Tensor):
args = (args, )
trace_graph, torch_out, inputs_states = \
torch.jit._get_trace_graph(func, args, strict=False, _force_outplace=False, _return_inputs_states=True)
warn_on_static_input_change(inputs_states)
trace_graph = _optimize_graph(trace_graph, operator_export_type, params_dict={})
if return_outs:
return trace_graph, torch_out
return trace_graph
def _trace_and_get_graph_from_model(model, args):
# A basic sanity check: make sure the state_dict keys are the same
# before and after running the model. Fail fast!
orig_state_dict_keys = _unique_state_dict(model).keys()
trace_graph, torch_out, inputs_states = \
torch.jit._get_trace_graph(model, args, strict=False, _force_outplace=False, _return_inputs_states=True)
warn_on_static_input_change(inputs_states)
if orig_state_dict_keys != _unique_state_dict(model).keys():
raise RuntimeError("state_dict changed after running the tracer; "
"something weird is happening in your model!")
return trace_graph, torch_out
def _get_param_count_list(method_graph, args_params):
param_count_list = []
for input_, arg_params_ in zip(method_graph.inputs(), args_params):
if "PackedParams" in str(input_.type()):
in_vars, _ = torch.jit._flatten(arg_params_)
param_count_list.append(len(in_vars))
else:
param_count_list.append(1)
return param_count_list
def _create_jit_graph(model, args):
torch_out = None
params: Union[List, Tuple]
if isinstance(model, torch.jit.ScriptModule):
try:
graph = model.forward.graph
torch._C._jit_pass_onnx_function_substitution(graph)
freezed_m = torch._C._freeze_module(model._c, preserveParameters=True)
module, params = torch._C._jit_onnx_list_model_parameters(freezed_m)
method_graph = module._get_method("forward").graph
args_params = tuple(args) + tuple(params)
param_count_list = _get_param_count_list(method_graph, args_params)
in_vars, _ = torch.jit._flatten(args_params)
graph = _propagate_and_assign_input_shapes(
method_graph, tuple(in_vars), param_count_list, False, False)
except AttributeError as e:
raise RuntimeError("'forward' method must be a script method") from e
return graph, params, torch_out, module
elif isinstance(model, torch.jit.ScriptFunction):
params = ()
in_vars, in_desc = torch.jit._flatten(tuple(args))
graph = model.graph
torch._C._jit_pass_onnx_function_substitution(graph)
param_count_list = _get_param_count_list(graph, args)
graph = _propagate_and_assign_input_shapes(
graph, tuple(in_vars), param_count_list, False, False)
return graph, params, torch_out, None
else:
graph, torch_out = _trace_and_get_graph_from_model(model, args)
torch._C._jit_pass_onnx_lint(graph)
state_dict = _unique_state_dict(model)
params = list(state_dict.values())
graph_inputs = list(graph.inputs())
user_input_num = len(graph_inputs) - len(state_dict)
param_names = list(state_dict.keys())
for i, inp in enumerate(graph_inputs):
if i >= user_input_num:
inp.setDebugName(param_names[i - user_input_num])
torch._C._jit_pass_onnx_function_substitution(graph)
return graph, params, torch_out, None
def _get_named_param_dict(graph, params):
input_and_param_names = [val.debugName() for val in graph.inputs()]
param_names = input_and_param_names[len(input_and_param_names) - len(params):]
_params_dict = dict(zip(param_names, params))
return _params_dict
def _get_example_outputs(model, args):
input_args = copy.deepcopy(args)
input_kwargs = {}
if input_args and isinstance(input_args[-1], dict):
input_kwargs = input_args[-1]
input_args = input_args[:-1]
example_outputs = model(*input_args, **input_kwargs)
if isinstance(example_outputs, (torch.Tensor, int, float, bool)):
example_outputs = (example_outputs,)
if isinstance(example_outputs, list):
example_outputs = [example_outputs]
return example_outputs
def _model_to_graph(model, args, verbose=False,
input_names=None, output_names=None,
operator_export_type=OperatorExportTypes.ONNX,
do_constant_folding=True,
_disable_torch_constant_prop=False, fixed_batch_size=False,
training=None, dynamic_axes=None):
r"""Converts model into an ONNX graph.
Returns:
graph (torch._C.Graph): A TorchScript IR Graph with ONNX nodes.
params_dict (Dict[str, torch.Tensor]): Dict from input param name to param value.
torch_out (Union[NoneType, torch.Tensor, Tuple[torch.Tensor], List[torch.Tensor]]):
The output tensors resulting from the trace of ``model``.
If ``model`` is a :class:`torch.jit.ScriptModule` or :class:`torch.jit.ScriptFunction`,
this will be None, since we are not doing any tracing.
"""
# TODO: can we simplify this to always return a tuple of Tensor or None?
from torch.onnx.symbolic_helper import _export_onnx_opset_version
# Special case for common case of passing a single Tensor
if isinstance(args, (torch.Tensor, int, float, bool)):
args = (args, )
graph, params, torch_out, module = _create_jit_graph(model, args)
params_dict = _get_named_param_dict(graph, params)
graph = _optimize_graph(graph, operator_export_type,
_disable_torch_constant_prop=_disable_torch_constant_prop,
fixed_batch_size=fixed_batch_size, params_dict=params_dict,
dynamic_axes=dynamic_axes, input_names=input_names,
module=module)
from torch.onnx.symbolic_helper import _onnx_shape_inference
if isinstance(model, torch.jit.ScriptModule) or isinstance(model, torch.jit.ScriptFunction):
example_outputs = _get_example_outputs(model, args)
out_vars, desc = torch.jit._flatten(tuple(example_outputs))
torch._C._jit_pass_onnx_assign_output_shape(graph, out_vars, desc, _onnx_shape_inference)
else:
flatten_args, _ = torch._C._jit_flatten(args)
# make sure that the param dict and the graph match each other
assert len(params) + len(flatten_args) == sum(1 for _ in graph.inputs())
# NB: ONNX requires complete information about output types, which might be
# erased by some optimizations, so we need to set it explicitly again.
if torch_out is not None:
if not (isinstance(torch_out, list) or isinstance(torch_out, tuple)):
output_wrapped = [torch_out]
else:
output_wrapped = torch_out # type: ignore[assignment]
output_tensors, out_desc = torch._C._jit_flatten(tuple(output_wrapped))
torch._C._jit_pass_onnx_assign_output_shape(graph, output_tensors, out_desc, _onnx_shape_inference)
_set_input_and_output_names(graph, input_names, output_names)
params_dict = _get_named_param_dict(graph, params)
if training is None or training == TrainingMode.EVAL:
params_dict = torch._C._jit_pass_onnx_eval_peephole(graph, params_dict)
if do_constant_folding and _export_onnx_opset_version in torch.onnx.constant_folding_opset_versions:
params_dict = torch._C._jit_pass_onnx_constant_fold(graph, params_dict,
_export_onnx_opset_version)
torch._C._jit_pass_dce_allow_deleting_nodes_with_side_effects(graph)
if _onnx_shape_inference:
torch._C._jit_pass_onnx_graph_shape_type_inference(graph, params_dict, _export_onnx_opset_version)
params_dict = torch._C._jit_pass_onnx_eliminate_unused_items(graph, params_dict)
# For ONNX opset < 9, constants only have three data types: float16, float, double.
# In this pass transform constants of other data types to float/double + cast operator.
if _export_onnx_opset_version < 9:
torch._C._jit_pass_onnx_cast_all_constant_to_floating(graph)
if verbose:
print(graph)
params_dict = torch._C._jit_pass_filter_non_tensor_arguments(params_dict)
torch._C._jit_decay_packed_param_input_types(graph)
return graph, params_dict, torch_out
def export_to_pretty_string(model, args, export_params=True, verbose=False, training=None,
input_names=None, output_names=None, operator_export_type=OperatorExportTypes.ONNX,
export_type=ExportTypes.PROTOBUF_FILE, google_printer=False, opset_version=None,
keep_initializers_as_inputs=None, custom_opsets=None, add_node_names=True,
do_constant_folding=True, dynamic_axes=None):
from torch.onnx.symbolic_helper import _default_onnx_opset_version, _set_opset_version
from torch.onnx.symbolic_helper import _set_operator_export_type
if opset_version is None:
opset_version = _default_onnx_opset_version
if custom_opsets is None:
custom_opsets = {}
_set_opset_version(opset_version)
_set_operator_export_type(operator_export_type)
from torch.onnx.symbolic_helper import _set_onnx_shape_inference
_set_onnx_shape_inference(True)
with exporter_context(model, training):
val_keep_init_as_ip = _decide_keep_init_as_input(keep_initializers_as_inputs,
operator_export_type,
opset_version)
val_add_node_names = _decide_add_node_names(add_node_names, operator_export_type)
val_do_constant_folding = _decide_constant_folding(do_constant_folding, operator_export_type, training)
args = _decide_input_format(model, args)
graph, params_dict, torch_out = _model_to_graph(model, args, verbose, input_names,
output_names, operator_export_type,
val_do_constant_folding,
training=training, dynamic_axes=dynamic_axes)
return graph._pretty_print_onnx(params_dict, opset_version, False,
operator_export_type, google_printer,
val_keep_init_as_ip, custom_opsets, val_add_node_names)
def unconvertible_ops(model, args, training=TrainingMode.EVAL, opset_version=None):
r"""
Converts the model with operator_export_type set to
OperatorExportTypes.ONNX_FALLTHROUGH once in order to get a list of
all the ops that are not supported/implemented by the exporter.
Args:
model: Same as corresponding arg to torch.onnx.export.
args: Same as corresponding arg to torch.onnx.export.
training: Same as corresponding arg to torch.onnx.export.
opset_version: Same as corresponding arg to torch.onnx.export.
Returns:
Tuple[torch._C.Graph, List[str]], where the list includes the names
of the unconvertible ops.
"""
from torch.onnx.symbolic_helper import _default_onnx_opset_version, _set_opset_version
opset_version = opset_version or _default_onnx_opset_version
_set_opset_version(opset_version)
# operator_export_type is set to ONNX_FALLTHROUGH by default so that if an op is not supported
# in ONNX, fall through will occur and export the operator as is, as a custom ONNX op.
operator_export_type = OperatorExportTypes.ONNX_FALLTHROUGH
with exporter_context(model, training):
args = _decide_input_format(model, args)
graph, params_dict, torch_out = _model_to_graph(
model, args,
# So that if an op connot be converted to ONNX, it will be kept
# as-is rather than cause a failure.
operator_export_type=OperatorExportTypes.ONNX_FALLTHROUGH)
unsupported_ops = list()
supported_namespaces = ("onnx", "prim")
for node in graph.nodes():
if node.kind().split(":")[0] not in supported_namespaces:
unsupported_ops.append(node.kind())
return graph, unsupported_ops
def _setup_trace_module_map(model, export_modules_as_functions):
def __setup_trace_module_map():
trace_module_map = {_m : torch.typename(type(_m)) for _m in model.modules()}
torch.jit._trace._trace_module_map = trace_module_map
return trace_module_map
if isinstance(export_modules_as_functions, bool) and export_modules_as_functions:
trace_module_map = __setup_trace_module_map()
export_modules_as_functions = {v for k, v in trace_module_map.items()}
elif isinstance(export_modules_as_functions, set) and len(export_modules_as_functions) > 0:
def _find_typename(v):
if isinstance(v, type):
return torch.typename(v)
else:
raise RuntimeError("Only type of the `nn.Module` should be "
"passed in the set for argument `export_modules_as_functions`. "
"Got `%s`." % (type(v).__name__))
trace_module_map = __setup_trace_module_map()
module_typenames = {_find_typename(v) for v in export_modules_as_functions}
export_modules_as_functions = module_typenames
else:
export_modules_as_functions = None
return export_modules_as_functions
def _reset_trace_module_map():
torch.jit._trace._trace_module_map = None
def _export(model, args, f, export_params=True, verbose=False, training=None,
input_names=None, output_names=None, operator_export_type=None,
export_type=ExportTypes.PROTOBUF_FILE, opset_version=None,
do_constant_folding=True, dynamic_axes=None, keep_initializers_as_inputs=None,
fixed_batch_size=False, custom_opsets=None, add_node_names=True,
onnx_shape_inference=True, export_modules_as_functions=False):
export_modules_as_functions = _setup_trace_module_map(model, export_modules_as_functions)
if isinstance(model, torch.nn.DataParallel):
raise ValueError("torch.nn.DataParallel is not supported by ONNX "
"exporter, please use 'attribute' module to "
"unwrap model from torch.nn.DataParallel. Try "
"torch.onnx.export(model.module, ...)")
global __IN_ONNX_EXPORT
assert __IN_ONNX_EXPORT is False
__IN_ONNX_EXPORT = True
try:
from torch.onnx.symbolic_helper import _set_onnx_shape_inference
_set_onnx_shape_inference(onnx_shape_inference)
from torch.onnx.symbolic_helper import _default_onnx_opset_version, _set_opset_version
from torch.onnx.symbolic_helper import _set_operator_export_type
if opset_version is None:
opset_version = _default_onnx_opset_version
if not operator_export_type:
if torch.onnx.PYTORCH_ONNX_CAFFE2_BUNDLE:
operator_export_type = OperatorExportTypes.ONNX_ATEN_FALLBACK
else:
operator_export_type = OperatorExportTypes.ONNX
# By default, training=None, (which defaults to TrainingMode.EVAL),
# which is good because running a model in training mode could result in
# internal buffers getting updated, dropout getting applied, etc.
# If you really know what you're doing, you can turn
# training=TrainingMode.TRAINING or training=TrainingMode.PRESERVE,
# (to preserve whatever the original training mode was.)
_set_opset_version(opset_version)
_set_operator_export_type(operator_export_type)
with exporter_context(model, training):
val_keep_init_as_ip = _decide_keep_init_as_input(keep_initializers_as_inputs,
operator_export_type,
opset_version)
val_add_node_names = _decide_add_node_names(add_node_names, operator_export_type)
val_do_constant_folding = _decide_constant_folding(do_constant_folding, operator_export_type, training)
# Normally f can be a file-like object, but for large models, the external data format requires a
# valid `model_file_location`. Code in export.cpp will enforce this.
if isinstance(f, str):
model_file_location = f
else:
model_file_location = str()
args = _decide_input_format(model, args)
if dynamic_axes is None:
dynamic_axes = {}
_validate_dynamic_axes(dynamic_axes, model, input_names, output_names)
graph, params_dict, torch_out = \
_model_to_graph(model, args, verbose, input_names,
output_names, operator_export_type,
val_do_constant_folding,
fixed_batch_size=fixed_batch_size,
training=training,
dynamic_axes=dynamic_axes)
# TODO: Don't allocate a in-memory string for the protobuf
defer_weight_export = export_type is not ExportTypes.PROTOBUF_FILE
if custom_opsets is None:
custom_opsets = {}
torch._C._jit_pass_dce_allow_deleting_nodes_with_side_effects(graph)
node_attr_to_name = {} # type: ignore[var-annotated]
if export_modules_as_functions is not None:
# NOTE: cannot call DCE after this pass. DCE will remove function definition nodes.
node_attr_to_name = torch._C._jit_pass_onnx_function_extraction(
graph, export_modules_as_functions, list(params_dict.keys()))
if export_params:
proto, export_map, val_use_external_data_format = graph._export_onnx(
params_dict, opset_version, dynamic_axes, defer_weight_export,
operator_export_type, not verbose, val_keep_init_as_ip, custom_opsets,
val_add_node_names, model_file_location, node_attr_to_name)
else:
proto, export_map, val_use_external_data_format = graph._export_onnx(
{}, opset_version, dynamic_axes, False, operator_export_type,
not verbose, val_keep_init_as_ip, custom_opsets, val_add_node_names,
model_file_location, node_attr_to_name)
if export_type == ExportTypes.PROTOBUF_FILE:
assert(len(export_map) == 0)
with torch.serialization._open_file_like(f, "wb") as opened_file:
opened_file.write(proto)
elif export_type in [ExportTypes.ZIP_ARCHIVE, ExportTypes.COMPRESSED_ZIP_ARCHIVE]:
import zipfile
compression = zipfile.ZIP_DEFLATED \
if export_type == ExportTypes.COMPRESSED_ZIP_ARCHIVE \
else zipfile.ZIP_STORED
with zipfile.ZipFile(f, "w", compression=compression) as z:
z.writestr(ONNX_ARCHIVE_MODEL_PROTO_NAME, proto)
for k, v in export_map.items():
z.writestr(k, v)
elif export_type == ExportTypes.DIRECTORY:
import os
if os.path.exists(f):
assert(os.path.isdir(f))
else:
os.makedirs(f)
model_proto_file = os.path.join(f, ONNX_ARCHIVE_MODEL_PROTO_NAME)
with torch.serialization._open_file_like(model_proto_file, "wb") as opened_file:
opened_file.write(proto)
for k, v in export_map.items():
weight_proto_file = os.path.join(f, k)
with torch.serialization._open_file_like(weight_proto_file, "wb") as opened_file:
opened_file.write(v)
else:
raise RuntimeError("Unknown export type")
# The ONNX checker only works for ONNX graph. So if the operator_export_type is not ONNX,
# we can skip this check.
# If large model format export is enabled, proto will only contain data location instead of
# raw data and _check_onnx_proto() will fail because it can only handle the raw ONNX proto
# string in memory.
if (operator_export_type is OperatorExportTypes.ONNX) and (not val_use_external_data_format):
try:
_check_onnx_proto(proto)
except RuntimeError as e:
raise CheckerError(e)
finally:
assert __IN_ONNX_EXPORT
__IN_ONNX_EXPORT = False
_reset_trace_module_map()
return torch_out
def _set_input_and_output_names(graph, input_names, output_names):
def set_names(node_list, name_list, descriptor):
if name_list is None:
return
if len(name_list) > len(node_list):
raise RuntimeError(
"number of %s names provided (%d) exceeded number of %ss (%d)"
% (descriptor, len(name_list), descriptor, len(node_list)))
# Mark if the output node DebugName is set before.
output_node_set = set()
for i, (name, node) in enumerate(zip(name_list, node_list)):
# Duplicated output node, insert onnx::Identity to avoid setting the same DebugName after setDebugName().
if descriptor == "output":
if node in output_node_set:
identity_node = graph.create("onnx::Identity")
identity_node.insertAfter(node.node())
identity_node.addInput(node)
identity_node.output().setType(node.type())
graph.return_node().replaceInput(i, identity_node.output())
node = identity_node.output()
output_node_set.add(node)
if node.debugName() != name:
node.setDebugName(name)
set_names(list(graph.inputs()), input_names, "input")
set_names(list(graph.outputs()), output_names, "output")
attr_pattern = re.compile("^(.+)_([ifstgz])$")
def _run_symbolic_method(g, op_name, symbolic_fn, args):
r"""
This trampoline function gets invoked for every symbolic method
call from C++.
"""
try:
return symbolic_fn(g, *args)
except TypeError as e:
# Handle the specific case where we didn't successfully dispatch
# to symbolic_fn. Otherwise, the backtrace will have the clues
# you need.
e.args = ("{} (occurred when translating {})".format(e.args[0], op_name),)
raise
def _is_onnx_list(value):