forked from codertimo/BERT-pytorch
-
Notifications
You must be signed in to change notification settings - Fork 12
/
Digraph.gv
2283 lines (2283 loc) · 71.9 KB
/
Digraph.gv
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
digraph {
graph [size="321.75,321.75"]
node [align=left fontsize=12 height=0.2 ranksep=0.1 shape=box style=filled]
4557578928 [label=LogSoftmaxBackward fillcolor=darkolivegreen1]
4676889848 -> 4557578928
4676889848 [label=AddmmBackward]
4676888840 -> 4676889848
4676888840 [label=ExpandBackward]
4557445328 -> 4676888840
4557445328 [label="next_sentence.linear.bias
(2)" fillcolor=lightblue]
4676887720 -> 4676889848
4676887720 [label=AsStridedBackward]
4574135328 -> 4676887720
4574135328 [label=DropoutBackward]
4676889120 -> 4574135328
4676889120 [label=AddBackward1]
4676887552 -> 4676889120
4676887552 [label=AddBackward1]
4662199656 -> 4676887552
4662199656 [label=DropoutBackward]
4676888952 -> 4662199656
4676888952 [label=AddBackward1]
4676889232 -> 4676888952
4676889232 [label=AddBackward1]
4662197856 -> 4676889232
4662197856 [label=DropoutBackward]
4676889288 -> 4662197856
4676889288 [label=AddBackward1]
4676889456 -> 4676889288
4676889456 [label=AddBackward1]
4662196216 -> 4676889456
4662196216 [label=DropoutBackward]
4662358480 -> 4662196216
4662358480 [label=AddBackward1]
4662358592 -> 4662358480
4662358592 [label=AddBackward1]
4662194576 -> 4662358592
4662194576 [label=DropoutBackward]
4662358704 -> 4662194576
4662358704 [label=AddBackward1]
4662394216 -> 4662358704
4662394216 [label=AddBackward1]
4662188680 -> 4662394216
4662188680 [label=DropoutBackward]
4662394664 -> 4662188680
4662394664 [label=AddBackward1]
4662394776 -> 4662394664
4662394776 [label=AddBackward1]
4662187040 -> 4662394776
4662187040 [label=DropoutBackward]
4662243456 -> 4662187040
4662243456 [label=AddBackward1]
4662243568 -> 4662243456
4662243568 [label=AddBackward1]
4661296408 -> 4662243568
4661296408 [label=DropoutBackward]
4662243736 -> 4661296408
4662243736 [label=AddBackward1]
4662243848 -> 4662243736
4662243848 [label=AddBackward1]
4661294440 -> 4662243848
4661294440 [label=DropoutBackward]
4662244016 -> 4661294440
4662244016 [label=AddBackward1]
4661295424 -> 4662244016
4661295424 [label=DropoutBackward]
4662244240 -> 4661295424
4662244240 [label=AddBackward1]
4662244352 -> 4662244240
4662244352 [label=ExpandBackward]
4662244464 -> 4662244352
4662244464 [label=EmbeddingBackward]
4557542792 -> 4662244464
4557542792 [label="bert.embedding.token.weight
(15, 256)" fillcolor=lightblue]
4662244128 -> 4662244016
4662244128 [label=EmbeddingBackward]
4557542400 -> 4662244128
4557542400 [label="bert.embedding.segment.weight
(3, 256)" fillcolor=lightblue]
4661294768 -> 4662243848
4661294768 [label=DropoutBackward]
4662244072 -> 4661294768
4662244072 [label=AddBackward1]
4662244296 -> 4662244072
4662244296 [label=UnsafeViewBackward]
4662244576 -> 4662244296
4662244576 [label=MmBackward]
4662244688 -> 4662244576
4662244688 [label=ViewBackward]
4662244856 -> 4662244688
4662244856 [label=ViewBackward]
4662244968 -> 4662244856
4662244968 [label=CloneBackward]
4662245080 -> 4662244968
4662245080 [label=TransposeBackward]
4662245192 -> 4662245080
4662245192 [label=UnsafeViewBackward]
4662245304 -> 4662245192
4662245304 [label=BmmBackward]
4662245416 -> 4662245304
4662245416 [label=ViewBackward]
4662245584 -> 4662245416
4662245584 [label=ExpandBackward]
4661295096 -> 4662245584
4661295096 [label=DropoutBackward]
4662245752 -> 4661295096
4662245752 [label=SoftmaxBackward]
4662245864 -> 4662245752
4662245864 [label=MaskedFillBackward0]
4662245976 -> 4662245864
4662245976 [label=CloneBackward]
4662246088 -> 4662245976
4662246088 [label=DivBackward0]
4662246200 -> 4662246088
4662246200 [label=UnsafeViewBackward]
4662246312 -> 4662246200
4662246312 [label=BmmBackward]
4662246424 -> 4662246312
4662246424 [label=ViewBackward]
4662246592 -> 4662246424
4662246592 [label=CloneBackward]
4662246704 -> 4662246592
4662246704 [label=ExpandBackward]
4662246816 -> 4662246704
4662246816 [label=TransposeBackward]
4662246928 -> 4662246816
4662246928 [label=ViewBackward]
4662247040 -> 4662246928
4662247040 [label=AddBackward1]
4662247152 -> 4662247040
4662247152 [label=UnsafeViewBackward]
4662247320 -> 4662247152
4662247320 [label=MmBackward]
4662259784 -> 4662247320
4662259784 [label=ViewBackward]
4662259952 -> 4662259784
4662259952 [label=AddBackward1]
4662260064 -> 4662259952
4662260064 [label=ExpandBackward]
4662260232 -> 4662260064
4662260232 [label=DivBackward1]
4662260344 -> 4662260232
4662260344 [label=ExpandBackward]
4662260512 -> 4662260344
4662260512 [label=MulBackward1]
4662260624 -> 4662260512
4662260624 [label=ExpandBackward]
4557556200 -> 4662260624
4557556200 [label="bert.transformer_blocks.0.input_sublayer.norm.a_2
(256)" fillcolor=lightblue]
4662260680 -> 4662260512
4662260680 [label=ExpandBackward]
4662260792 -> 4662260680
4662260792 [label=SubBackward1]
4662260904 -> 4662260792
4662260904 [label=ExpandBackward]
4661294440 -> 4662260904
4662260960 -> 4662260792
4662260960 [label=ExpandBackward]
4662261072 -> 4662260960
4662261072 [label=MeanBackward0]
4661294440 -> 4662261072
4662260400 -> 4662260232
4662260400 [label=ExpandBackward]
4662260568 -> 4662260400
4662260568 [label=AddBackward0]
4662260848 -> 4662260568
4662260848 [label=StdBackward1]
4661294440 -> 4662260848
4662260120 -> 4662259952
4662260120 [label=ExpandBackward]
4557555584 -> 4662260120
4557555584 [label="bert.transformer_blocks.0.input_sublayer.norm.b_2
(256)" fillcolor=lightblue]
4662259840 -> 4662247320
4662259840 [label=TBackward]
4557555248 -> 4662259840
4557555248 [label="bert.transformer_blocks.0.attention.linear_layers.0.weight
(256, 256)" fillcolor=lightblue]
4662247208 -> 4662247040
4662247208 [label=ExpandBackward]
4557554912 -> 4662247208
4557554912 [label="bert.transformer_blocks.0.attention.linear_layers.0.bias
(256)" fillcolor=lightblue]
4662246480 -> 4662246312
4662246480 [label=ViewBackward]
4662246648 -> 4662246480
4662246648 [label=CloneBackward]
4662246872 -> 4662246648
4662246872 [label=ExpandBackward]
4662247096 -> 4662246872
4662247096 [label=TransposeBackward]
4662247376 -> 4662247096
4662247376 [label=TransposeBackward]
4662260008 -> 4662247376
4662260008 [label=ViewBackward]
4662260456 -> 4662260008
4662260456 [label=AddBackward1]
4662260736 -> 4662260456
4662260736 [label=UnsafeViewBackward]
4662261184 -> 4662260736
4662261184 [label=MmBackward]
4662261296 -> 4662261184
4662261296 [label=ViewBackward]
4662259952 -> 4662261296
4662261352 -> 4662261184
4662261352 [label=TBackward]
4557557264 -> 4662261352
4557557264 [label="bert.transformer_blocks.0.attention.linear_layers.1.weight
(256, 256)" fillcolor=lightblue]
4662261016 -> 4662260456
4662261016 [label=ExpandBackward]
4557557040 -> 4662261016
4557557040 [label="bert.transformer_blocks.0.attention.linear_layers.1.bias
(256)" fillcolor=lightblue]
4662245472 -> 4662245304
4662245472 [label=ViewBackward]
4662245640 -> 4662245472
4662245640 [label=CloneBackward]
4662245808 -> 4662245640
4662245808 [label=ExpandBackward]
4662246032 -> 4662245808
4662246032 [label=TransposeBackward]
4662246256 -> 4662246032
4662246256 [label=ViewBackward]
4662246536 -> 4662246256
4662246536 [label=AddBackward1]
4662246984 -> 4662246536
4662246984 [label=UnsafeViewBackward]
4662260288 -> 4662246984
4662260288 [label=MmBackward]
4662261128 -> 4662260288
4662261128 [label=ViewBackward]
4662259952 -> 4662261128
4662261240 -> 4662260288
4662261240 [label=TBackward]
4557557432 -> 4662261240
4557557432 [label="bert.transformer_blocks.0.attention.linear_layers.2.weight
(256, 256)" fillcolor=lightblue]
4662247264 -> 4662246536
4662247264 [label=ExpandBackward]
4557556704 -> 4662247264
4557556704 [label="bert.transformer_blocks.0.attention.linear_layers.2.bias
(256)" fillcolor=lightblue]
4662244744 -> 4662244576
4662244744 [label=TBackward]
4557543296 -> 4662244744
4557543296 [label="bert.transformer_blocks.0.attention.output_linear.weight
(256, 256)" fillcolor=lightblue]
4662244520 -> 4662244072
4662244520 [label=ExpandBackward]
4557543016 -> 4662244520
4557543016 [label="bert.transformer_blocks.0.attention.output_linear.bias
(256)" fillcolor=lightblue]
4661296080 -> 4662243736
4661296080 [label=DropoutBackward]
4662243960 -> 4661296080
4662243960 [label=AddBackward1]
4662244408 -> 4662243960
4662244408 [label=UnsafeViewBackward]
4662244912 -> 4662244408
4662244912 [label=MmBackward]
4662245248 -> 4662244912
4662245248 [label=ViewBackward]
4661295752 -> 4662245248
4661295752 [label=DropoutBackward]
4662245696 -> 4661295752
4662245696 [label=MulBackward1]
4662246144 -> 4662245696
4662246144 [label=MulBackward0]
4662259896 -> 4662246144
4662259896 [label=AddBackward1]
4662261408 -> 4662259896
4662261408 [label=UnsafeViewBackward]
4662261520 -> 4662261408
4662261520 [label=MmBackward]
4662261688 -> 4662261520
4662261688 [label=ViewBackward]
4662261856 -> 4662261688
4662261856 [label=AddBackward1]
4662261968 -> 4662261856
4662261968 [label=ExpandBackward]
4662262136 -> 4662261968
4662262136 [label=DivBackward1]
4662262248 -> 4662262136
4662262248 [label=ExpandBackward]
4662262416 -> 4662262248
4662262416 [label=MulBackward1]
4662262528 -> 4662262416
4662262528 [label=ExpandBackward]
4557579376 -> 4662262528
4557579376 [label="bert.transformer_blocks.0.output_sublayer.norm.a_2
(256)" fillcolor=lightblue]
4662262584 -> 4662262416
4662262584 [label=ExpandBackward]
4662262696 -> 4662262584
4662262696 [label=SubBackward1]
4662262808 -> 4662262696
4662262808 [label=ExpandBackward]
4662243848 -> 4662262808
4662262864 -> 4662262696
4662262864 [label=ExpandBackward]
4662262976 -> 4662262864
4662262976 [label=MeanBackward0]
4662243848 -> 4662262976
4662262304 -> 4662262136
4662262304 [label=ExpandBackward]
4662262472 -> 4662262304
4662262472 [label=AddBackward0]
4662262752 -> 4662262472
4662262752 [label=StdBackward1]
4662243848 -> 4662262752
4662262024 -> 4662261856
4662262024 [label=ExpandBackward]
4557578872 -> 4662262024
4557578872 [label="bert.transformer_blocks.0.output_sublayer.norm.b_2
(256)" fillcolor=lightblue]
4662261744 -> 4662261520
4662261744 [label=TBackward]
4557578592 -> 4662261744
4557578592 [label="bert.transformer_blocks.0.feed_forward.w_1.weight
(1024, 256)" fillcolor=lightblue]
4662261464 -> 4662259896
4662261464 [label=ExpandBackward]
4557578312 -> 4662261464
4557578312 [label="bert.transformer_blocks.0.feed_forward.w_1.bias
(1024)" fillcolor=lightblue]
4662246368 -> 4662245696
4662246368 [label=AddBackward0]
4662260176 -> 4662246368
4662260176 [label=TanhBackward]
4662261576 -> 4662260176
4662261576 [label=MulBackward0]
4662261912 -> 4662261576
4662261912 [label=AddBackward1]
4662259896 -> 4662261912
4662262360 -> 4662261912
4662262360 [label=MulBackward0]
4662262640 -> 4662262360
4662262640 [label=PowBackward0]
4662259896 -> 4662262640
4662245024 -> 4662244912
4662245024 [label=TBackward]
4557544640 -> 4662245024
4557544640 [label="bert.transformer_blocks.0.feed_forward.w_2.weight
(256, 1024)" fillcolor=lightblue]
4662244632 -> 4662243960
4662244632 [label=ExpandBackward]
4557543856 -> 4662244632
4557543856 [label="bert.transformer_blocks.0.feed_forward.w_2.bias
(256)" fillcolor=lightblue]
4662186056 -> 4662243568
4662186056 [label=DropoutBackward]
4662243792 -> 4662186056
4662243792 [label=AddBackward1]
4662244184 -> 4662243792
4662244184 [label=UnsafeViewBackward]
4662245360 -> 4662244184
4662245360 [label=MmBackward]
4662246760 -> 4662245360
4662246760 [label=ViewBackward]
4662261800 -> 4662246760
4662261800 [label=ViewBackward]
4662262080 -> 4662261800
4662262080 [label=CloneBackward]
4662263032 -> 4662262080
4662263032 [label=TransposeBackward]
4662263144 -> 4662263032
4662263144 [label=UnsafeViewBackward]
4662263256 -> 4662263144
4662263256 [label=BmmBackward]
4662263368 -> 4662263256
4662263368 [label=ViewBackward]
4662263536 -> 4662263368
4662263536 [label=ExpandBackward]
4661296736 -> 4662263536
4661296736 [label=DropoutBackward]
4662263704 -> 4661296736
4662263704 [label=SoftmaxBackward]
4662280264 -> 4662263704
4662280264 [label=MaskedFillBackward0]
4662280376 -> 4662280264
4662280376 [label=CloneBackward]
4662280488 -> 4662280376
4662280488 [label=DivBackward0]
4662280600 -> 4662280488
4662280600 [label=UnsafeViewBackward]
4662280712 -> 4662280600
4662280712 [label=BmmBackward]
4662280824 -> 4662280712
4662280824 [label=ViewBackward]
4662280992 -> 4662280824
4662280992 [label=CloneBackward]
4662281104 -> 4662280992
4662281104 [label=ExpandBackward]
4662281216 -> 4662281104
4662281216 [label=TransposeBackward]
4662281328 -> 4662281216
4662281328 [label=ViewBackward]
4662281440 -> 4662281328
4662281440 [label=AddBackward1]
4662281552 -> 4662281440
4662281552 [label=UnsafeViewBackward]
4662281720 -> 4662281552
4662281720 [label=MmBackward]
4662281832 -> 4662281720
4662281832 [label=ViewBackward]
4662282000 -> 4662281832
4662282000 [label=AddBackward1]
4662282112 -> 4662282000
4662282112 [label=ExpandBackward]
4662282280 -> 4662282112
4662282280 [label=DivBackward1]
4662282392 -> 4662282280
4662282392 [label=ExpandBackward]
4662282560 -> 4662282392
4662282560 [label=MulBackward1]
4662282672 -> 4662282560
4662282672 [label=ExpandBackward]
4557591664 -> 4662282672
4557591664 [label="bert.transformer_blocks.1.input_sublayer.norm.a_2
(256)" fillcolor=lightblue]
4662282728 -> 4662282560
4662282728 [label=ExpandBackward]
4662282840 -> 4662282728
4662282840 [label=SubBackward1]
4662282952 -> 4662282840
4662282952 [label=ExpandBackward]
4661296408 -> 4662282952
4662283008 -> 4662282840
4662283008 [label=ExpandBackward]
4662283120 -> 4662283008
4662283120 [label=MeanBackward0]
4661296408 -> 4662283120
4662282448 -> 4662282280
4662282448 [label=ExpandBackward]
4662282616 -> 4662282448
4662282616 [label=AddBackward0]
4662282896 -> 4662282616
4662282896 [label=StdBackward1]
4661296408 -> 4662282896
4662282168 -> 4662282000
4662282168 [label=ExpandBackward]
4557591160 -> 4662282168
4557591160 [label="bert.transformer_blocks.1.input_sublayer.norm.b_2
(256)" fillcolor=lightblue]
4662281888 -> 4662281720
4662281888 [label=TBackward]
4557590880 -> 4662281888
4557590880 [label="bert.transformer_blocks.1.attention.linear_layers.0.weight
(256, 256)" fillcolor=lightblue]
4662281608 -> 4662281440
4662281608 [label=ExpandBackward]
4557590600 -> 4662281608
4557590600 [label="bert.transformer_blocks.1.attention.linear_layers.0.bias
(256)" fillcolor=lightblue]
4662280880 -> 4662280712
4662280880 [label=ViewBackward]
4662281048 -> 4662280880
4662281048 [label=CloneBackward]
4662281272 -> 4662281048
4662281272 [label=ExpandBackward]
4662281496 -> 4662281272
4662281496 [label=TransposeBackward]
4662281776 -> 4662281496
4662281776 [label=TransposeBackward]
4662282056 -> 4662281776
4662282056 [label=ViewBackward]
4662282504 -> 4662282056
4662282504 [label=AddBackward1]
4662282784 -> 4662282504
4662282784 [label=UnsafeViewBackward]
4662283232 -> 4662282784
4662283232 [label=MmBackward]
4662283344 -> 4662283232
4662283344 [label=ViewBackward]
4662282000 -> 4662283344
4662283400 -> 4662283232
4662283400 [label=TBackward]
4557592560 -> 4662283400
4557592560 [label="bert.transformer_blocks.1.attention.linear_layers.1.weight
(256, 256)" fillcolor=lightblue]
4662283064 -> 4662282504
4662283064 [label=ExpandBackward]
4557592336 -> 4662283064
4557592336 [label="bert.transformer_blocks.1.attention.linear_layers.1.bias
(256)" fillcolor=lightblue]
4662263424 -> 4662263256
4662263424 [label=ViewBackward]
4662263592 -> 4662263424
4662263592 [label=CloneBackward]
4662263760 -> 4662263592
4662263760 [label=ExpandBackward]
4662280432 -> 4662263760
4662280432 [label=TransposeBackward]
4662280656 -> 4662280432
4662280656 [label=ViewBackward]
4662280936 -> 4662280656
4662280936 [label=AddBackward1]
4662281384 -> 4662280936
4662281384 [label=UnsafeViewBackward]
4662282336 -> 4662281384
4662282336 [label=MmBackward]
4662283176 -> 4662282336
4662283176 [label=ViewBackward]
4662282000 -> 4662283176
4662283288 -> 4662282336
4662283288 [label=TBackward]
4557592728 -> 4662283288
4557592728 [label="bert.transformer_blocks.1.attention.linear_layers.2.weight
(256, 256)" fillcolor=lightblue]
4662281664 -> 4662280936
4662281664 [label=ExpandBackward]
4557592056 -> 4662281664
4557592056 [label="bert.transformer_blocks.1.attention.linear_layers.2.bias
(256)" fillcolor=lightblue]
4662245920 -> 4662245360
4662245920 [label=TBackward]
4557579880 -> 4662245920
4557579880 [label="bert.transformer_blocks.1.attention.output_linear.weight
(256, 256)" fillcolor=lightblue]
4662244800 -> 4662243792
4662244800 [label=ExpandBackward]
4557557656 -> 4662244800
4557557656 [label="bert.transformer_blocks.1.attention.output_linear.bias
(256)" fillcolor=lightblue]
4662186712 -> 4662243456
4662186712 [label=DropoutBackward]
4662243680 -> 4662186712
4662243680 [label=AddBackward1]
4662245136 -> 4662243680
4662245136 [label=UnsafeViewBackward]
4662262192 -> 4662245136
4662262192 [label=MmBackward]
4662263200 -> 4662262192
4662263200 [label=ViewBackward]
4662186384 -> 4662263200
4662186384 [label=DropoutBackward]
4662263648 -> 4662186384
4662263648 [label=MulBackward1]
4662280544 -> 4662263648
4662280544 [label=MulBackward0]
4662281944 -> 4662280544
4662281944 [label=AddBackward1]
4662283456 -> 4662281944
4662283456 [label=UnsafeViewBackward]
4662283568 -> 4662283456
4662283568 [label=MmBackward]
4662283736 -> 4662283568
4662283736 [label=ViewBackward]
4662283904 -> 4662283736
4662283904 [label=AddBackward1]
4662284016 -> 4662283904
4662284016 [label=ExpandBackward]
4662284184 -> 4662284016
4662284184 [label=DivBackward1]
4662317128 -> 4662284184
4662317128 [label=ExpandBackward]
4662317296 -> 4662317128
4662317296 [label=MulBackward1]
4662317408 -> 4662317296
4662317408 [label=ExpandBackward]
4557594128 -> 4662317408
4557594128 [label="bert.transformer_blocks.1.output_sublayer.norm.a_2
(256)" fillcolor=lightblue]
4662317464 -> 4662317296
4662317464 [label=ExpandBackward]
4662317576 -> 4662317464
4662317576 [label=SubBackward1]
4662317688 -> 4662317576
4662317688 [label=ExpandBackward]
4662243568 -> 4662317688
4662317744 -> 4662317576
4662317744 [label=ExpandBackward]
4662317856 -> 4662317744
4662317856 [label=MeanBackward0]
4662243568 -> 4662317856
4662317184 -> 4662284184
4662317184 [label=ExpandBackward]
4662317352 -> 4662317184
4662317352 [label=AddBackward0]
4662317632 -> 4662317352
4662317632 [label=StdBackward1]
4662243568 -> 4662317632
4662284072 -> 4662283904
4662284072 [label=ExpandBackward]
4557593624 -> 4662284072
4557593624 [label="bert.transformer_blocks.1.output_sublayer.norm.b_2
(256)" fillcolor=lightblue]
4662283792 -> 4662283568
4662283792 [label=TBackward]
4557593344 -> 4662283792
4557593344 [label="bert.transformer_blocks.1.feed_forward.w_1.weight
(1024, 256)" fillcolor=lightblue]
4662283512 -> 4662281944
4662283512 [label=ExpandBackward]
4557593064 -> 4662283512
4557593064 [label="bert.transformer_blocks.1.feed_forward.w_1.bias
(1024)" fillcolor=lightblue]
4662280768 -> 4662263648
4662280768 [label=AddBackward0]
4662282224 -> 4662280768
4662282224 [label=TanhBackward]
4662283624 -> 4662282224
4662283624 [label=MulBackward0]
4662283960 -> 4662283624
4662283960 [label=AddBackward1]
4662281944 -> 4662283960
4662284128 -> 4662283960
4662284128 [label=MulBackward0]
4662317520 -> 4662284128
4662317520 [label=PowBackward0]
4662281944 -> 4662317520
4662262920 -> 4662262192
4662262920 [label=TBackward]
4557580888 -> 4662262920
4557580888 [label="bert.transformer_blocks.1.feed_forward.w_2.weight
(256, 1024)" fillcolor=lightblue]
4662245528 -> 4662243680
4662245528 [label=ExpandBackward]
4557580328 -> 4662245528
4557580328 [label="bert.transformer_blocks.1.feed_forward.w_2.bias
(256)" fillcolor=lightblue]
4662187696 -> 4662394776
4662187696 [label=DropoutBackward]
4662243512 -> 4662187696
4662243512 [label=AddBackward1]
4662243904 -> 4662243512
4662243904 [label=UnsafeViewBackward]
4662263312 -> 4662243904
4662263312 [label=MmBackward]
4662281160 -> 4662263312
4662281160 [label=ViewBackward]
4662283848 -> 4662281160
4662283848 [label=ViewBackward]
4662317240 -> 4662283848
4662317240 [label=CloneBackward]
4662317912 -> 4662317240
4662317912 [label=TransposeBackward]
4662318024 -> 4662317912
4662318024 [label=UnsafeViewBackward]
4662318136 -> 4662318024
4662318136 [label=BmmBackward]
4662318248 -> 4662318136
4662318248 [label=ViewBackward]
4662318416 -> 4662318248
4662318416 [label=ExpandBackward]
4662187368 -> 4662318416
4662187368 [label=DropoutBackward]
4662318584 -> 4662187368
4662318584 [label=SoftmaxBackward]
4662318696 -> 4662318584
4662318696 [label=MaskedFillBackward0]
4662318808 -> 4662318696
4662318808 [label=CloneBackward]
4662318920 -> 4662318808
4662318920 [label=DivBackward0]
4662319032 -> 4662318920
4662319032 [label=UnsafeViewBackward]
4662319144 -> 4662319032
4662319144 [label=BmmBackward]
4662319256 -> 4662319144
4662319256 [label=ViewBackward]
4662319424 -> 4662319256
4662319424 [label=CloneBackward]
4662319536 -> 4662319424
4662319536 [label=ExpandBackward]
4662319648 -> 4662319536
4662319648 [label=TransposeBackward]
4662319760 -> 4662319648
4662319760 [label=ViewBackward]
4662319872 -> 4662319760
4662319872 [label=AddBackward1]
4662319984 -> 4662319872
4662319984 [label=UnsafeViewBackward]
4662320152 -> 4662319984
4662320152 [label=MmBackward]
4662320264 -> 4662320152
4662320264 [label=ViewBackward]
4662320432 -> 4662320264
4662320432 [label=AddBackward1]
4662320544 -> 4662320432
4662320544 [label=ExpandBackward]
4662320712 -> 4662320544
4662320712 [label=DivBackward1]
4662320824 -> 4662320712
4662320824 [label=ExpandBackward]
4662320992 -> 4662320824
4662320992 [label=MulBackward1]
4662321104 -> 4662320992
4662321104 [label=ExpandBackward]
4557618704 -> 4662321104
4557618704 [label="bert.transformer_blocks.2.input_sublayer.norm.a_2
(256)" fillcolor=lightblue]
4662337608 -> 4662320992
4662337608 [label=ExpandBackward]
4662337720 -> 4662337608
4662337720 [label=SubBackward1]
4662337832 -> 4662337720
4662337832 [label=ExpandBackward]
4662187040 -> 4662337832
4662337888 -> 4662337720
4662337888 [label=ExpandBackward]
4662338000 -> 4662337888
4662338000 [label=MeanBackward0]
4662187040 -> 4662338000
4662320880 -> 4662320712
4662320880 [label=ExpandBackward]
4662321048 -> 4662320880
4662321048 [label=AddBackward0]
4662337776 -> 4662321048
4662337776 [label=StdBackward1]
4662187040 -> 4662337776
4662320600 -> 4662320432
4662320600 [label=ExpandBackward]
4557618200 -> 4662320600
4557618200 [label="bert.transformer_blocks.2.input_sublayer.norm.b_2
(256)" fillcolor=lightblue]
4662320320 -> 4662320152
4662320320 [label=TBackward]
4557617920 -> 4662320320
4557617920 [label="bert.transformer_blocks.2.attention.linear_layers.0.weight
(256, 256)" fillcolor=lightblue]
4662320040 -> 4662319872
4662320040 [label=ExpandBackward]
4557617640 -> 4662320040
4557617640 [label="bert.transformer_blocks.2.attention.linear_layers.0.bias
(256)" fillcolor=lightblue]
4662319312 -> 4662319144
4662319312 [label=ViewBackward]
4662319480 -> 4662319312
4662319480 [label=CloneBackward]
4662319704 -> 4662319480
4662319704 [label=ExpandBackward]
4662319928 -> 4662319704
4662319928 [label=TransposeBackward]
4662320208 -> 4662319928
4662320208 [label=TransposeBackward]
4662320488 -> 4662320208
4662320488 [label=ViewBackward]
4662320936 -> 4662320488
4662320936 [label=AddBackward1]
4662337664 -> 4662320936
4662337664 [label=UnsafeViewBackward]
4662338112 -> 4662337664
4662338112 [label=MmBackward]
4662338224 -> 4662338112
4662338224 [label=ViewBackward]
4662320432 -> 4662338224
4662338280 -> 4662338112
4662338280 [label=TBackward]
4557623760 -> 4662338280
4557623760 [label="bert.transformer_blocks.2.attention.linear_layers.1.weight
(256, 256)" fillcolor=lightblue]
4662337944 -> 4662320936
4662337944 [label=ExpandBackward]
4557623536 -> 4662337944
4557623536 [label="bert.transformer_blocks.2.attention.linear_layers.1.bias
(256)" fillcolor=lightblue]
4662318304 -> 4662318136
4662318304 [label=ViewBackward]
4662318472 -> 4662318304
4662318472 [label=CloneBackward]
4662318640 -> 4662318472
4662318640 [label=ExpandBackward]
4662318864 -> 4662318640
4662318864 [label=TransposeBackward]
4662319088 -> 4662318864
4662319088 [label=ViewBackward]
4662319368 -> 4662319088
4662319368 [label=AddBackward1]
4662319816 -> 4662319368
4662319816 [label=UnsafeViewBackward]
4662320768 -> 4662319816
4662320768 [label=MmBackward]
4662338056 -> 4662320768
4662338056 [label=ViewBackward]
4662320432 -> 4662338056
4662338168 -> 4662320768
4662338168 [label=TBackward]
4557623928 -> 4662338168
4557623928 [label="bert.transformer_blocks.2.attention.linear_layers.2.weight
(256, 256)" fillcolor=lightblue]
4662320096 -> 4662319368
4662320096 [label=ExpandBackward]
4557619096 -> 4662320096
4557619096 [label="bert.transformer_blocks.2.attention.linear_layers.2.bias
(256)" fillcolor=lightblue]
4662283680 -> 4662263312
4662283680 [label=TBackward]
4557615176 -> 4662283680
4557615176 [label="bert.transformer_blocks.2.attention.output_linear.weight
(256, 256)" fillcolor=lightblue]
4662261632 -> 4662243512
4662261632 [label=ExpandBackward]
4557592952 -> 4662261632
4557592952 [label="bert.transformer_blocks.2.attention.output_linear.bias
(256)" fillcolor=lightblue]
4662188352 -> 4662394664
4662188352 [label=DropoutBackward]
4662243400 -> 4662188352
4662243400 [label=AddBackward1]
4662263088 -> 4662243400
4662263088 [label=UnsafeViewBackward]
4662284240 -> 4662263088
4662284240 [label=MmBackward]
4662318080 -> 4662284240
4662318080 [label=ViewBackward]
4662188024 -> 4662318080
4662188024 [label=DropoutBackward]
4662318528 -> 4662188024
4662318528 [label=MulBackward1]
4662318976 -> 4662318528
4662318976 [label=MulBackward0]
4662320376 -> 4662318976
4662320376 [label=AddBackward1]
4662338336 -> 4662320376
4662338336 [label=UnsafeViewBackward]
4662338448 -> 4662338336
4662338448 [label=MmBackward]
4662338616 -> 4662338448
4662338616 [label=ViewBackward]
4662338784 -> 4662338616
4662338784 [label=AddBackward1]
4662338896 -> 4662338784
4662338896 [label=ExpandBackward]
4662339064 -> 4662338896
4662339064 [label=DivBackward1]
4662339176 -> 4662339064
4662339176 [label=ExpandBackward]
4662339344 -> 4662339176
4662339344 [label=MulBackward1]
4662339456 -> 4662339344
4662339456 [label=ExpandBackward]
4557625328 -> 4662339456
4557625328 [label="bert.transformer_blocks.2.output_sublayer.norm.a_2
(256)" fillcolor=lightblue]
4662339512 -> 4662339344
4662339512 [label=ExpandBackward]
4662339624 -> 4662339512
4662339624 [label=SubBackward1]
4662339736 -> 4662339624
4662339736 [label=ExpandBackward]
4662394776 -> 4662339736
4662339792 -> 4662339624
4662339792 [label=ExpandBackward]
4662339904 -> 4662339792
4662339904 [label=MeanBackward0]
4662394776 -> 4662339904
4662339232 -> 4662339064
4662339232 [label=ExpandBackward]
4662339400 -> 4662339232
4662339400 [label=AddBackward0]
4662339680 -> 4662339400
4662339680 [label=StdBackward1]
4662394776 -> 4662339680
4662338952 -> 4662338784
4662338952 [label=ExpandBackward]
4557624824 -> 4662338952
4557624824 [label="bert.transformer_blocks.2.output_sublayer.norm.b_2
(256)" fillcolor=lightblue]
4662338672 -> 4662338448
4662338672 [label=TBackward]
4557624544 -> 4662338672
4557624544 [label="bert.transformer_blocks.2.feed_forward.w_1.weight
(1024, 256)" fillcolor=lightblue]
4662338392 -> 4662320376
4662338392 [label=ExpandBackward]
4557624264 -> 4662338392
4557624264 [label="bert.transformer_blocks.2.feed_forward.w_1.bias
(1024)" fillcolor=lightblue]
4662319200 -> 4662318528
4662319200 [label=AddBackward0]
4662320656 -> 4662319200
4662320656 [label=TanhBackward]
4662338504 -> 4662320656
4662338504 [label=MulBackward0]
4662338840 -> 4662338504
4662338840 [label=AddBackward1]
4662320376 -> 4662338840
4662339288 -> 4662338840
4662339288 [label=MulBackward0]
4662339568 -> 4662339288
4662339568 [label=PowBackward0]
4662320376 -> 4662339568
4662317800 -> 4662284240
4662317800 [label=TBackward]
4557616184 -> 4662317800
4557616184 [label="bert.transformer_blocks.2.feed_forward.w_2.weight
(256, 1024)" fillcolor=lightblue]
4662263480 -> 4662243400
4662263480 [label=ExpandBackward]
4557615624 -> 4662263480
4557615624 [label="bert.transformer_blocks.2.feed_forward.w_2.bias
(256)" fillcolor=lightblue]
4662189336 -> 4662394216
4662189336 [label=DropoutBackward]
4662394720 -> 4662189336
4662394720 [label=AddBackward1]
4662243624 -> 4662394720
4662243624 [label=UnsafeViewBackward]
4662318192 -> 4662243624
4662318192 [label=MmBackward]
4662319592 -> 4662318192
4662319592 [label=ViewBackward]
4662338728 -> 4662319592
4662338728 [label=ViewBackward]
4662339008 -> 4662338728
4662339008 [label=CloneBackward]
4662339960 -> 4662339008
4662339960 [label=TransposeBackward]
4662340072 -> 4662339960
4662340072 [label=UnsafeViewBackward]
4662340184 -> 4662340072
4662340184 [label=BmmBackward]
4662340296 -> 4662340184
4662340296 [label=ViewBackward]
4662340464 -> 4662340296
4662340464 [label=ExpandBackward]
4662189008 -> 4662340464
4662189008 [label=DropoutBackward]
4662340632 -> 4662189008
4662340632 [label=SoftmaxBackward]
4662340744 -> 4662340632
4662340744 [label=MaskedFillBackward0]
4662340856 -> 4662340744
4662340856 [label=CloneBackward]
4662340968 -> 4662340856
4662340968 [label=DivBackward0]
4662341080 -> 4662340968
4662341080 [label=UnsafeViewBackward]
4662341192 -> 4662341080
4662341192 [label=BmmBackward]
4662341304 -> 4662341192
4662341304 [label=ViewBackward]
4662341472 -> 4662341304
4662341472 [label=CloneBackward]
4662341584 -> 4662341472
4662341584 [label=ExpandBackward]
4662366336 -> 4662341584
4662366336 [label=TransposeBackward]
4662366448 -> 4662366336
4662366448 [label=ViewBackward]
4662366560 -> 4662366448
4662366560 [label=AddBackward1]
4662366672 -> 4662366560
4662366672 [label=UnsafeViewBackward]
4662366840 -> 4662366672
4662366840 [label=MmBackward]
4662366952 -> 4662366840
4662366952 [label=ViewBackward]
4662367120 -> 4662366952
4662367120 [label=AddBackward1]
4662367232 -> 4662367120
4662367232 [label=ExpandBackward]
4662367400 -> 4662367232
4662367400 [label=DivBackward1]
4662367512 -> 4662367400
4662367512 [label=ExpandBackward]
4662367680 -> 4662367512
4662367680 [label=MulBackward1]
4662367792 -> 4662367680
4662367792 [label=ExpandBackward]
4486570448 -> 4662367792
4486570448 [label="bert.transformer_blocks.3.input_sublayer.norm.a_2
(256)" fillcolor=lightblue]
4662367848 -> 4662367680
4662367848 [label=ExpandBackward]
4662367960 -> 4662367848
4662367960 [label=SubBackward1]
4662368072 -> 4662367960
4662368072 [label=ExpandBackward]
4662188680 -> 4662368072
4662368128 -> 4662367960
4662368128 [label=ExpandBackward]
4662368240 -> 4662368128
4662368240 [label=MeanBackward0]
4662188680 -> 4662368240
4662367568 -> 4662367400
4662367568 [label=ExpandBackward]
4662367736 -> 4662367568
4662367736 [label=AddBackward0]
4662368016 -> 4662367736
4662368016 [label=StdBackward1]
4662188680 -> 4662368016
4662367288 -> 4662367120
4662367288 [label=ExpandBackward]
4656308968 -> 4662367288
4656308968 [label="bert.transformer_blocks.3.input_sublayer.norm.b_2
(256)" fillcolor=lightblue]
4662367008 -> 4662366840
4662367008 [label=TBackward]
4656308520 -> 4662367008
4656308520 [label="bert.transformer_blocks.3.attention.linear_layers.0.weight
(256, 256)" fillcolor=lightblue]
4662366728 -> 4662366560
4662366728 [label=ExpandBackward]
4656309472 -> 4662366728