mirrored from git://gcc.gnu.org/git/gcc.git
/
internal-fn.c
2825 lines (2498 loc) · 85.4 KB
/
internal-fn.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/* Internal functions.
Copyright (C) 2011-2017 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.
GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
<http://www.gnu.org/licenses/>. */
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "target.h"
#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "predict.h"
#include "stringpool.h"
#include "tree-vrp.h"
#include "tree-ssanames.h"
#include "expmed.h"
#include "memmodel.h"
#include "optabs.h"
#include "emit-rtl.h"
#include "diagnostic-core.h"
#include "fold-const.h"
#include "internal-fn.h"
#include "stor-layout.h"
#include "dojump.h"
#include "expr.h"
#include "stringpool.h"
#include "attribs.h"
#include "asan.h"
#include "ubsan.h"
#include "recog.h"
#include "builtins.h"
#include "optabs-tree.h"
/* The names of each internal function, indexed by function number. */
const char *const internal_fn_name_array[] = {
#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
#include "internal-fn.def"
"<invalid-fn>"
};
/* The ECF_* flags of each internal function, indexed by function number. */
const int internal_fn_flags_array[] = {
#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
#include "internal-fn.def"
0
};
/* Fnspec of each internal function, indexed by function number. */
const_tree internal_fn_fnspec_array[IFN_LAST + 1];
void
init_internal_fns ()
{
#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
#include "internal-fn.def"
internal_fn_fnspec_array[IFN_LAST] = 0;
}
/* Create static initializers for the information returned by
direct_internal_fn. */
#define not_direct { -2, -2, false }
#define mask_load_direct { -1, 2, false }
#define load_lanes_direct { -1, -1, false }
#define mask_store_direct { 3, 2, false }
#define store_lanes_direct { 0, 0, false }
#define unary_direct { 0, 0, true }
#define binary_direct { 0, 0, true }
const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
#define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
#include "internal-fn.def"
not_direct
};
/* ARRAY_TYPE is an array of vector modes. Return the associated insn
for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
static enum insn_code
get_multi_vector_move (tree array_type, convert_optab optab)
{
machine_mode imode;
machine_mode vmode;
gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
imode = TYPE_MODE (array_type);
vmode = TYPE_MODE (TREE_TYPE (array_type));
return convert_optab_handler (optab, imode, vmode);
}
/* Expand LOAD_LANES call STMT using optab OPTAB. */
static void
expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
{
struct expand_operand ops[2];
tree type, lhs, rhs;
rtx target, mem;
lhs = gimple_call_lhs (stmt);
rhs = gimple_call_arg (stmt, 0);
type = TREE_TYPE (lhs);
target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
mem = expand_normal (rhs);
gcc_assert (MEM_P (mem));
PUT_MODE (mem, TYPE_MODE (type));
create_output_operand (&ops[0], target, TYPE_MODE (type));
create_fixed_operand (&ops[1], mem);
expand_insn (get_multi_vector_move (type, optab), 2, ops);
}
/* Expand STORE_LANES call STMT using optab OPTAB. */
static void
expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
{
struct expand_operand ops[2];
tree type, lhs, rhs;
rtx target, reg;
lhs = gimple_call_lhs (stmt);
rhs = gimple_call_arg (stmt, 0);
type = TREE_TYPE (rhs);
target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
reg = expand_normal (rhs);
gcc_assert (MEM_P (target));
PUT_MODE (target, TYPE_MODE (type));
create_fixed_operand (&ops[0], target);
create_input_operand (&ops[1], reg, TYPE_MODE (type));
expand_insn (get_multi_vector_move (type, optab), 2, ops);
}
static void
expand_ANNOTATE (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in omp_device_lower pass. */
static void
expand_GOMP_USE_SIMT (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in omp_device_lower pass. */
static void
expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* Allocate per-lane storage and begin non-uniform execution region. */
static void
expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
{
rtx target;
tree lhs = gimple_call_lhs (stmt);
if (lhs)
target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
else
target = gen_reg_rtx (Pmode);
rtx size = expand_normal (gimple_call_arg (stmt, 0));
rtx align = expand_normal (gimple_call_arg (stmt, 1));
struct expand_operand ops[3];
create_output_operand (&ops[0], target, Pmode);
create_input_operand (&ops[1], size, Pmode);
create_input_operand (&ops[2], align, Pmode);
gcc_assert (targetm.have_omp_simt_enter ());
expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
}
/* Deallocate per-lane storage and leave non-uniform execution region. */
static void
expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
{
gcc_checking_assert (!gimple_call_lhs (stmt));
rtx arg = expand_normal (gimple_call_arg (stmt, 0));
struct expand_operand ops[1];
create_input_operand (&ops[0], arg, Pmode);
gcc_assert (targetm.have_omp_simt_exit ());
expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
}
/* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
without SIMT execution this should be expanded in omp_device_lower pass. */
static void
expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
{
tree lhs = gimple_call_lhs (stmt);
if (!lhs)
return;
rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
gcc_assert (targetm.have_omp_simt_lane ());
emit_insn (targetm.gen_omp_simt_lane (target));
}
/* This should get expanded in omp_device_lower pass. */
static void
expand_GOMP_SIMT_VF (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* Lane index of the first SIMT lane that supplies a non-zero argument.
This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
lane that executed the last iteration for handling OpenMP lastprivate. */
static void
expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
{
tree lhs = gimple_call_lhs (stmt);
if (!lhs)
return;
rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
rtx cond = expand_normal (gimple_call_arg (stmt, 0));
machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
struct expand_operand ops[2];
create_output_operand (&ops[0], target, mode);
create_input_operand (&ops[1], cond, mode);
gcc_assert (targetm.have_omp_simt_last_lane ());
expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
}
/* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
static void
expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
{
tree lhs = gimple_call_lhs (stmt);
if (!lhs)
return;
rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
struct expand_operand ops[2];
create_output_operand (&ops[0], target, mode);
create_input_operand (&ops[1], ctr, mode);
gcc_assert (targetm.have_omp_simt_ordered ());
expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
}
/* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
any lane supplies a non-zero argument. */
static void
expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
{
tree lhs = gimple_call_lhs (stmt);
if (!lhs)
return;
rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
rtx cond = expand_normal (gimple_call_arg (stmt, 0));
machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
struct expand_operand ops[2];
create_output_operand (&ops[0], target, mode);
create_input_operand (&ops[1], cond, mode);
gcc_assert (targetm.have_omp_simt_vote_any ());
expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
}
/* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
is destination lane index XOR given offset. */
static void
expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
{
tree lhs = gimple_call_lhs (stmt);
if (!lhs)
return;
rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
rtx src = expand_normal (gimple_call_arg (stmt, 0));
rtx idx = expand_normal (gimple_call_arg (stmt, 1));
machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
struct expand_operand ops[3];
create_output_operand (&ops[0], target, mode);
create_input_operand (&ops[1], src, mode);
create_input_operand (&ops[2], idx, SImode);
gcc_assert (targetm.have_omp_simt_xchg_bfly ());
expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
}
/* Exchange between SIMT lanes according to given source lane index. */
static void
expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
{
tree lhs = gimple_call_lhs (stmt);
if (!lhs)
return;
rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
rtx src = expand_normal (gimple_call_arg (stmt, 0));
rtx idx = expand_normal (gimple_call_arg (stmt, 1));
machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
struct expand_operand ops[3];
create_output_operand (&ops[0], target, mode);
create_input_operand (&ops[1], src, mode);
create_input_operand (&ops[2], idx, SImode);
gcc_assert (targetm.have_omp_simt_xchg_idx ());
expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
}
/* This should get expanded in adjust_simduid_builtins. */
static void
expand_GOMP_SIMD_LANE (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in adjust_simduid_builtins. */
static void
expand_GOMP_SIMD_VF (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in adjust_simduid_builtins. */
static void
expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in adjust_simduid_builtins. */
static void
expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in adjust_simduid_builtins. */
static void
expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_UBSAN_NULL (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_UBSAN_BOUNDS (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_UBSAN_VPTR (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_UBSAN_PTR (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_ASAN_CHECK (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_ASAN_MARK (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_ASAN_POISON (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the sanopt pass. */
static void
expand_ASAN_POISON_USE (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the tsan pass. */
static void
expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
{
gcc_unreachable ();
}
/* This should get expanded in the lower pass. */
static void
expand_FALLTHROUGH (internal_fn, gcall *call)
{
error_at (gimple_location (call),
"invalid use of attribute %<fallthrough%>");
}
/* Return minimum precision needed to represent all values
of ARG in SIGNed integral type. */
static int
get_min_precision (tree arg, signop sign)
{
int prec = TYPE_PRECISION (TREE_TYPE (arg));
int cnt = 0;
signop orig_sign = sign;
if (TREE_CODE (arg) == INTEGER_CST)
{
int p;
if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
{
widest_int w = wi::to_widest (arg);
w = wi::ext (w, prec, sign);
p = wi::min_precision (w, sign);
}
else
p = wi::min_precision (wi::to_wide (arg), sign);
return MIN (p, prec);
}
while (CONVERT_EXPR_P (arg)
&& INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
&& TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
{
arg = TREE_OPERAND (arg, 0);
if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
{
if (TYPE_UNSIGNED (TREE_TYPE (arg)))
sign = UNSIGNED;
else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
return prec + (orig_sign != sign);
prec = TYPE_PRECISION (TREE_TYPE (arg));
}
if (++cnt > 30)
return prec + (orig_sign != sign);
}
if (TREE_CODE (arg) != SSA_NAME)
return prec + (orig_sign != sign);
wide_int arg_min, arg_max;
while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
{
gimple *g = SSA_NAME_DEF_STMT (arg);
if (is_gimple_assign (g)
&& CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
{
tree t = gimple_assign_rhs1 (g);
if (INTEGRAL_TYPE_P (TREE_TYPE (t))
&& TYPE_PRECISION (TREE_TYPE (t)) <= prec)
{
arg = t;
if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
{
if (TYPE_UNSIGNED (TREE_TYPE (arg)))
sign = UNSIGNED;
else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
return prec + (orig_sign != sign);
prec = TYPE_PRECISION (TREE_TYPE (arg));
}
if (++cnt > 30)
return prec + (orig_sign != sign);
continue;
}
}
return prec + (orig_sign != sign);
}
if (sign == TYPE_SIGN (TREE_TYPE (arg)))
{
int p1 = wi::min_precision (arg_min, sign);
int p2 = wi::min_precision (arg_max, sign);
p1 = MAX (p1, p2);
prec = MIN (prec, p1);
}
else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
{
int p = wi::min_precision (arg_max, UNSIGNED);
prec = MIN (prec, p);
}
return prec + (orig_sign != sign);
}
/* Helper for expand_*_overflow. Set the __imag__ part to true
(1 except for signed:1 type, in which case store -1). */
static void
expand_arith_set_overflow (tree lhs, rtx target)
{
if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
&& !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
write_complex_part (target, constm1_rtx, true);
else
write_complex_part (target, const1_rtx, true);
}
/* Helper for expand_*_overflow. Store RES into the __real__ part
of TARGET. If RES has larger MODE than __real__ part of TARGET,
set the __imag__ part to 1 if RES doesn't fit into it. Similarly
if LHS has smaller precision than its mode. */
static void
expand_arith_overflow_result_store (tree lhs, rtx target,
scalar_int_mode mode, rtx res)
{
scalar_int_mode tgtmode
= as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
rtx lres = res;
if (tgtmode != mode)
{
rtx_code_label *done_label = gen_label_rtx ();
int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
lres = convert_modes (tgtmode, mode, res, uns);
gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
EQ, true, mode, NULL_RTX, NULL, done_label,
profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
emit_label (done_label);
}
int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
int tgtprec = GET_MODE_PRECISION (tgtmode);
if (prec < tgtprec)
{
rtx_code_label *done_label = gen_label_rtx ();
int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
res = lres;
if (uns)
{
rtx mask
= immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
tgtmode);
lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
true, OPTAB_LIB_WIDEN);
}
else
{
lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
NULL_RTX, 1);
lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
NULL_RTX, 0);
}
do_compare_rtx_and_jump (res, lres,
EQ, true, tgtmode, NULL_RTX, NULL, done_label,
profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
emit_label (done_label);
}
write_complex_part (target, lres, false);
}
/* Helper for expand_*_overflow. Store RES into TARGET. */
static void
expand_ubsan_result_store (rtx target, rtx res)
{
if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
/* If this is a scalar in a register that is stored in a wider mode
than the declared mode, compute the result into its declared mode
and then convert to the wider mode. Our value is the computed
expression. */
convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
else
emit_move_insn (target, res);
}
/* Add sub/add overflow checking to the statement STMT.
CODE says whether the operation is +, or -. */
static void
expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
tree arg0, tree arg1, bool unsr_p, bool uns0_p,
bool uns1_p, bool is_ubsan, tree *datap)
{
rtx res, target = NULL_RTX;
tree fn;
rtx_code_label *done_label = gen_label_rtx ();
rtx_code_label *do_error = gen_label_rtx ();
do_pending_stack_adjust ();
rtx op0 = expand_normal (arg0);
rtx op1 = expand_normal (arg1);
scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
int prec = GET_MODE_PRECISION (mode);
rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
bool do_xor = false;
if (is_ubsan)
gcc_assert (!unsr_p && !uns0_p && !uns1_p);
if (lhs)
{
target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
if (!is_ubsan)
write_complex_part (target, const0_rtx, true);
}
/* We assume both operands and result have the same precision
here (GET_MODE_BITSIZE (mode)), S stands for signed type
with that precision, U for unsigned type with that precision,
sgn for unsigned most significant bit in that precision.
s1 is signed first operand, u1 is unsigned first operand,
s2 is signed second operand, u2 is unsigned second operand,
sr is signed result, ur is unsigned result and the following
rules say how to compute result (which is always result of
the operands as if both were unsigned, cast to the right
signedness) and how to compute whether operation overflowed.
s1 + s2 -> sr
res = (S) ((U) s1 + (U) s2)
ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
s1 - s2 -> sr
res = (S) ((U) s1 - (U) s2)
ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
u1 + u2 -> ur
res = u1 + u2
ovf = res < u1 (or jump on carry, but RTL opts will handle it)
u1 - u2 -> ur
res = u1 - u2
ovf = res > u1 (or jump on carry, but RTL opts will handle it)
s1 + u2 -> sr
res = (S) ((U) s1 + u2)
ovf = ((U) res ^ sgn) < u2
s1 + u2 -> ur
t1 = (S) (u2 ^ sgn)
t2 = s1 + t1
res = (U) t2 ^ sgn
ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
s1 - u2 -> sr
res = (S) ((U) s1 - u2)
ovf = u2 > ((U) s1 ^ sgn)
s1 - u2 -> ur
res = (U) s1 - u2
ovf = s1 < 0 || u2 > (U) s1
u1 - s2 -> sr
res = u1 - (U) s2
ovf = u1 >= ((U) s2 ^ sgn)
u1 - s2 -> ur
t1 = u1 ^ sgn
t2 = t1 - (U) s2
res = t2 ^ sgn
ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
s1 + s2 -> ur
res = (U) s1 + (U) s2
ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
u1 + u2 -> sr
res = (S) (u1 + u2)
ovf = (U) res < u2 || res < 0
u1 - u2 -> sr
res = (S) (u1 - u2)
ovf = u1 >= u2 ? res < 0 : res >= 0
s1 - s2 -> ur
res = (U) s1 - (U) s2
ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
if (code == PLUS_EXPR && uns0_p && !uns1_p)
{
/* PLUS_EXPR is commutative, if operand signedness differs,
canonicalize to the first operand being signed and second
unsigned to simplify following code. */
std::swap (op0, op1);
std::swap (arg0, arg1);
uns0_p = false;
uns1_p = true;
}
/* u1 +- u2 -> ur */
if (uns0_p && uns1_p && unsr_p)
{
insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
: usubv4_optab, mode);
if (icode != CODE_FOR_nothing)
{
struct expand_operand ops[4];
rtx_insn *last = get_last_insn ();
res = gen_reg_rtx (mode);
create_output_operand (&ops[0], res, mode);
create_input_operand (&ops[1], op0, mode);
create_input_operand (&ops[2], op1, mode);
create_fixed_operand (&ops[3], do_error);
if (maybe_expand_insn (icode, 4, ops))
{
last = get_last_insn ();
if (profile_status_for_fn (cfun) != PROFILE_ABSENT
&& JUMP_P (last)
&& any_condjump_p (last)
&& !find_reg_note (last, REG_BR_PROB, 0))
add_reg_br_prob_note (last,
profile_probability::very_unlikely ());
emit_jump (done_label);
goto do_error_label;
}
delete_insns_since (last);
}
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
rtx tem = op0;
/* For PLUS_EXPR, the operation is commutative, so we can pick
operand to compare against. For prec <= BITS_PER_WORD, I think
preferring REG operand is better over CONST_INT, because
the CONST_INT might enlarge the instruction or CSE would need
to figure out we'd already loaded it into a register before.
For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
as then the multi-word comparison can be perhaps simplified. */
if (code == PLUS_EXPR
&& (prec <= BITS_PER_WORD
? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
: CONST_SCALAR_INT_P (op1)))
tem = op1;
do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
true, mode, NULL_RTX, NULL, done_label,
profile_probability::very_likely ());
goto do_error_label;
}
/* s1 +- u2 -> sr */
if (!uns0_p && uns1_p && !unsr_p)
{
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
rtx tem = expand_binop (mode, add_optab,
code == PLUS_EXPR ? res : op0, sgn,
NULL_RTX, false, OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
done_label, profile_probability::very_likely ());
goto do_error_label;
}
/* s1 + u2 -> ur */
if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
{
op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
OPTAB_LIB_WIDEN);
/* As we've changed op1, we have to avoid using the value range
for the original argument. */
arg1 = error_mark_node;
do_xor = true;
goto do_signed;
}
/* u1 - s2 -> ur */
if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
{
op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
OPTAB_LIB_WIDEN);
/* As we've changed op0, we have to avoid using the value range
for the original argument. */
arg0 = error_mark_node;
do_xor = true;
goto do_signed;
}
/* s1 - u2 -> ur */
if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
{
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
OPTAB_LIB_WIDEN);
int pos_neg = get_range_pos_neg (arg0);
if (pos_neg == 2)
/* If ARG0 is known to be always negative, this is always overflow. */
emit_jump (do_error);
else if (pos_neg == 3)
/* If ARG0 is not known to be always positive, check at runtime. */
do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
NULL, do_error, profile_probability::very_unlikely ());
do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
done_label, profile_probability::very_likely ());
goto do_error_label;
}
/* u1 - s2 -> sr */
if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
{
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
OPTAB_LIB_WIDEN);
rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
done_label, profile_probability::very_likely ());
goto do_error_label;
}
/* u1 + u2 -> sr */
if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
{
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
NULL, do_error, profile_probability::very_unlikely ());
rtx tem = op1;
/* The operation is commutative, so we can pick operand to compare
against. For prec <= BITS_PER_WORD, I think preferring REG operand
is better over CONST_INT, because the CONST_INT might enlarge the
instruction or CSE would need to figure out we'd already loaded it
into a register before. For prec > BITS_PER_WORD, I think CONST_INT
might be more beneficial, as then the multi-word comparison can be
perhaps simplified. */
if (prec <= BITS_PER_WORD
? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
: CONST_SCALAR_INT_P (op0))
tem = op0;
do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
done_label, profile_probability::very_likely ());
goto do_error_label;
}
/* s1 +- s2 -> ur */
if (!uns0_p && !uns1_p && unsr_p)
{
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
int pos_neg = get_range_pos_neg (arg1);
if (code == PLUS_EXPR)
{
int pos_neg0 = get_range_pos_neg (arg0);
if (pos_neg0 != 3 && pos_neg == 3)
{
std::swap (op0, op1);
pos_neg = pos_neg0;
}
}
rtx tem;
if (pos_neg != 3)
{
tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
? and_optab : ior_optab,
op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
NULL, done_label, profile_probability::very_likely ());
}
else
{
rtx_code_label *do_ior_label = gen_label_rtx ();
do_compare_rtx_and_jump (op1, const0_rtx,
code == MINUS_EXPR ? GE : LT, false, mode,
NULL_RTX, NULL, do_ior_label,
profile_probability::even ());
tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
NULL, done_label, profile_probability::very_likely ());
emit_jump (do_error);
emit_label (do_ior_label);
tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
NULL, done_label, profile_probability::very_likely ());
}
goto do_error_label;
}
/* u1 - u2 -> sr */
if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
{
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
OPTAB_LIB_WIDEN);
rtx_code_label *op0_geu_op1 = gen_label_rtx ();
do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
op0_geu_op1, profile_probability::even ());
do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
NULL, done_label, profile_probability::very_likely ());
emit_jump (do_error);
emit_label (op0_geu_op1);
do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
NULL, done_label, profile_probability::very_likely ());
goto do_error_label;
}
gcc_assert (!uns0_p && !uns1_p && !unsr_p);
/* s1 +- s2 -> sr */
do_signed:
{
insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
: subv4_optab, mode);
if (icode != CODE_FOR_nothing)
{
struct expand_operand ops[4];
rtx_insn *last = get_last_insn ();
res = gen_reg_rtx (mode);
create_output_operand (&ops[0], res, mode);
create_input_operand (&ops[1], op0, mode);
create_input_operand (&ops[2], op1, mode);
create_fixed_operand (&ops[3], do_error);
if (maybe_expand_insn (icode, 4, ops))
{
last = get_last_insn ();
if (profile_status_for_fn (cfun) != PROFILE_ABSENT
&& JUMP_P (last)
&& any_condjump_p (last)
&& !find_reg_note (last, REG_BR_PROB, 0))
add_reg_br_prob_note (last,
profile_probability::very_unlikely ());
emit_jump (done_label);
goto do_error_label;
}
delete_insns_since (last);
}
/* Compute the operation. On RTL level, the addition is always
unsigned. */
res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
/* If we can prove that one of the arguments (for MINUS_EXPR only
the second operand, as subtraction is not commutative) is always
non-negative or always negative, we can do just one comparison
and conditional jump. */
int pos_neg = get_range_pos_neg (arg1);
if (code == PLUS_EXPR)
{
int pos_neg0 = get_range_pos_neg (arg0);
if (pos_neg0 != 3 && pos_neg == 3)
{
std::swap (op0, op1);