/
TargetLowering.h
4567 lines (3953 loc) · 196 KB
/
TargetLowering.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//===- llvm/CodeGen/TargetLowering.h - Target Lowering Info -----*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file describes how to lower LLVM code to machine code. This has two
/// main components:
///
/// 1. Which ValueTypes are natively supported by the target.
/// 2. Which operations are supported for supported ValueTypes.
/// 3. Cost thresholds for alternative implementations of certain operations.
///
/// In addition it has a few other components, like information about FP
/// immediates.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_CODEGEN_TARGETLOWERING_H
#define LLVM_CODEGEN_TARGETLOWERING_H
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Alignment.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MachineValueType.h"
#include <algorithm>
#include <cassert>
#include <climits>
#include <cstdint>
#include <iterator>
#include <map>
#include <string>
#include <utility>
#include <vector>
namespace llvm {
class BranchProbability;
class CCState;
class CCValAssign;
class Constant;
class FastISel;
class FunctionLoweringInfo;
class GlobalValue;
class GISelKnownBits;
class IntrinsicInst;
struct KnownBits;
class LegacyDivergenceAnalysis;
class LLVMContext;
class MachineBasicBlock;
class MachineFunction;
class MachineInstr;
class MachineJumpTableInfo;
class MachineLoop;
class MachineRegisterInfo;
class MCContext;
class MCExpr;
class Module;
class ProfileSummaryInfo;
class TargetLibraryInfo;
class TargetMachine;
class TargetRegisterClass;
class TargetRegisterInfo;
class TargetTransformInfo;
class Value;
namespace Sched {
enum Preference {
None, // No preference
Source, // Follow source order.
RegPressure, // Scheduling for lowest register pressure.
Hybrid, // Scheduling for both latency and register pressure.
ILP, // Scheduling for ILP in low register pressure mode.
VLIW // Scheduling for VLIW targets.
};
} // end namespace Sched
// MemOp models a memory operation, either memset or memcpy/memmove.
struct MemOp {
private:
// Shared
uint64_t Size;
bool DstAlignCanChange; // true if destination alignment can satisfy any
// constraint.
Align DstAlign; // Specified alignment of the memory operation.
bool AllowOverlap;
// memset only
bool IsMemset; // If setthis memory operation is a memset.
bool ZeroMemset; // If set clears out memory with zeros.
// memcpy only
bool MemcpyStrSrc; // Indicates whether the memcpy source is an in-register
// constant so it does not need to be loaded.
Align SrcAlign; // Inferred alignment of the source or default value if the
// memory operation does not need to load the value.
public:
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
Align SrcAlign, bool IsVolatile,
bool MemcpyStrSrc = false) {
MemOp Op;
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
Op.AllowOverlap = !IsVolatile;
Op.IsMemset = false;
Op.ZeroMemset = false;
Op.MemcpyStrSrc = MemcpyStrSrc;
Op.SrcAlign = SrcAlign;
return Op;
}
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign,
bool IsZeroMemset, bool IsVolatile) {
MemOp Op;
Op.Size = Size;
Op.DstAlignCanChange = DstAlignCanChange;
Op.DstAlign = DstAlign;
Op.AllowOverlap = !IsVolatile;
Op.IsMemset = true;
Op.ZeroMemset = IsZeroMemset;
Op.MemcpyStrSrc = false;
return Op;
}
uint64_t size() const { return Size; }
Align getDstAlign() const {
assert(!DstAlignCanChange);
return DstAlign;
}
bool isFixedDstAlign() const { return !DstAlignCanChange; }
bool allowOverlap() const { return AllowOverlap; }
bool isMemset() const { return IsMemset; }
bool isMemcpy() const { return !IsMemset; }
bool isMemcpyWithFixedDstAlign() const {
return isMemcpy() && !DstAlignCanChange;
}
bool isZeroMemset() const { return isMemset() && ZeroMemset; }
bool isMemcpyStrSrc() const {
assert(isMemcpy() && "Must be a memcpy");
return MemcpyStrSrc;
}
Align getSrcAlign() const {
assert(isMemcpy() && "Must be a memcpy");
return SrcAlign;
}
bool isSrcAligned(Align AlignCheck) const {
return isMemset() || llvm::isAligned(AlignCheck, SrcAlign.value());
}
bool isDstAligned(Align AlignCheck) const {
return DstAlignCanChange || llvm::isAligned(AlignCheck, DstAlign.value());
}
bool isAligned(Align AlignCheck) const {
return isSrcAligned(AlignCheck) && isDstAligned(AlignCheck);
}
};
/// This base class for TargetLowering contains the SelectionDAG-independent
/// parts that can be used from the rest of CodeGen.
class TargetLoweringBase {
public:
/// This enum indicates whether operations are valid for a target, and if not,
/// what action should be used to make them valid.
enum LegalizeAction : uint8_t {
Legal, // The target natively supports this operation.
Promote, // This operation should be executed in a larger type.
Expand, // Try to expand this to other ops, otherwise use a libcall.
LibCall, // Don't try to expand this to other ops, always use a libcall.
Custom // Use the LowerOperation hook to implement custom lowering.
};
/// This enum indicates whether a types are legal for a target, and if not,
/// what action should be used to make them valid.
enum LegalizeTypeAction : uint8_t {
TypeLegal, // The target natively supports this type.
TypePromoteInteger, // Replace this integer with a larger one.
TypeExpandInteger, // Split this integer into two of half the size.
TypeSoftenFloat, // Convert this float to a same size integer type.
TypeExpandFloat, // Split this float into two of half the size.
TypeScalarizeVector, // Replace this one-element vector with its element.
TypeSplitVector, // Split this vector into two of half the size.
TypeWidenVector, // This vector should be widened into a larger vector.
TypePromoteFloat, // Replace this float with a larger one.
TypeSoftPromoteHalf, // Soften half to i16 and use float to do arithmetic.
TypeScalarizeScalableVector, // This action is explicitly left unimplemented.
// While it is theoretically possible to
// legalize operations on scalable types with a
// loop that handles the vscale * #lanes of the
// vector, this is non-trivial at SelectionDAG
// level and these types are better to be
// widened or promoted.
};
/// LegalizeKind holds the legalization kind that needs to happen to EVT
/// in order to type-legalize it.
using LegalizeKind = std::pair<LegalizeTypeAction, EVT>;
/// Enum that describes how the target represents true/false values.
enum BooleanContent {
UndefinedBooleanContent, // Only bit 0 counts, the rest can hold garbage.
ZeroOrOneBooleanContent, // All bits zero except for bit 0.
ZeroOrNegativeOneBooleanContent // All bits equal to bit 0.
};
/// Enum that describes what type of support for selects the target has.
enum SelectSupportKind {
ScalarValSelect, // The target supports scalar selects (ex: cmov).
ScalarCondVectorVal, // The target supports selects with a scalar condition
// and vector values (ex: cmov).
VectorMaskSelect // The target supports vector selects with a vector
// mask (ex: x86 blends).
};
/// Enum that specifies what an atomic load/AtomicRMWInst is expanded
/// to, if at all. Exists because different targets have different levels of
/// support for these atomic instructions, and also have different options
/// w.r.t. what they should expand to.
enum class AtomicExpansionKind {
None, // Don't expand the instruction.
LLSC, // Expand the instruction into loadlinked/storeconditional; used
// by ARM/AArch64.
LLOnly, // Expand the (load) instruction into just a load-linked, which has
// greater atomic guarantees than a normal load.
CmpXChg, // Expand the instruction into cmpxchg; used by at least X86.
MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop.
};
/// Enum that specifies when a multiplication should be expanded.
enum class MulExpansionKind {
Always, // Always expand the instruction.
OnlyLegalOrCustom, // Only expand when the resulting instructions are legal
// or custom.
};
/// Enum that specifies when a float negation is beneficial.
enum class NegatibleCost {
Cheaper = 0, // Negated expression is cheaper.
Neutral = 1, // Negated expression has the same cost.
Expensive = 2 // Negated expression is more expensive.
};
class ArgListEntry {
public:
Value *Val = nullptr;
SDValue Node = SDValue();
Type *Ty = nullptr;
bool IsSExt : 1;
bool IsZExt : 1;
bool IsInReg : 1;
bool IsSRet : 1;
bool IsNest : 1;
bool IsByVal : 1;
bool IsByRef : 1;
bool IsInAlloca : 1;
bool IsPreallocated : 1;
bool IsReturned : 1;
bool IsSwiftSelf : 1;
bool IsSwiftError : 1;
bool IsCFGuardTarget : 1;
MaybeAlign Alignment = None;
Type *ByValType = nullptr;
Type *PreallocatedType = nullptr;
ArgListEntry()
: IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false),
IsNest(false), IsByVal(false), IsByRef(false), IsInAlloca(false),
IsPreallocated(false), IsReturned(false), IsSwiftSelf(false),
IsSwiftError(false), IsCFGuardTarget(false) {}
void setAttributes(const CallBase *Call, unsigned ArgIdx);
};
using ArgListTy = std::vector<ArgListEntry>;
virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC,
ArgListTy &Args) const {};
static ISD::NodeType getExtendForContent(BooleanContent Content) {
switch (Content) {
case UndefinedBooleanContent:
// Extend by adding rubbish bits.
return ISD::ANY_EXTEND;
case ZeroOrOneBooleanContent:
// Extend by adding zero bits.
return ISD::ZERO_EXTEND;
case ZeroOrNegativeOneBooleanContent:
// Extend by copying the sign bit.
return ISD::SIGN_EXTEND;
}
llvm_unreachable("Invalid content kind");
}
explicit TargetLoweringBase(const TargetMachine &TM);
TargetLoweringBase(const TargetLoweringBase &) = delete;
TargetLoweringBase &operator=(const TargetLoweringBase &) = delete;
virtual ~TargetLoweringBase() = default;
/// Return true if the target support strict float operation
bool isStrictFPEnabled() const {
return IsStrictFPEnabled;
}
protected:
/// Initialize all of the actions to default values.
void initActions();
public:
const TargetMachine &getTargetMachine() const { return TM; }
virtual bool useSoftFloat() const { return false; }
/// Return the pointer type for the given address space, defaults to
/// the pointer type from the data layout.
/// FIXME: The default needs to be removed once all the code is updated.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const {
return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
}
/// Return the in-memory pointer type for the given address space, defaults to
/// the pointer type from the data layout. FIXME: The default needs to be
/// removed once all the code is updated.
MVT getPointerMemTy(const DataLayout &DL, uint32_t AS = 0) const {
return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
}
/// Return the type for frame index, which is determined by
/// the alloca address space specified through the data layout.
MVT getFrameIndexTy(const DataLayout &DL) const {
return getPointerTy(DL, DL.getAllocaAddrSpace());
}
/// Return the type for code pointers, which is determined by the program
/// address space specified through the data layout.
MVT getProgramPointerTy(const DataLayout &DL) const {
return getPointerTy(DL, DL.getProgramAddressSpace());
}
/// Return the type for operands of fence.
/// TODO: Let fence operands be of i32 type and remove this.
virtual MVT getFenceOperandTy(const DataLayout &DL) const {
return getPointerTy(DL);
}
/// EVT is not used in-tree, but is used by out-of-tree target.
/// A documentation for this function would be nice...
virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const;
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL,
bool LegalTypes = true) const;
/// Return the preferred type to use for a shift opcode, given the shifted
/// amount type is \p ShiftValueTy.
LLVM_READONLY
virtual LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const {
return ShiftValueTy;
}
/// Returns the type to be used for the index operand of:
/// ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
/// ISD::INSERT_SUBVECTOR, and ISD::EXTRACT_SUBVECTOR
virtual MVT getVectorIdxTy(const DataLayout &DL) const {
return getPointerTy(DL);
}
/// This callback is used to inspect load/store instructions and add
/// target-specific MachineMemOperand flags to them. The default
/// implementation does nothing.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const {
return MachineMemOperand::MONone;
}
MachineMemOperand::Flags getLoadMemOperandFlags(const LoadInst &LI,
const DataLayout &DL) const;
MachineMemOperand::Flags getStoreMemOperandFlags(const StoreInst &SI,
const DataLayout &DL) const;
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI,
const DataLayout &DL) const;
virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
return true;
}
/// Return true if it is profitable to convert a select of FP constants into
/// a constant pool load whose address depends on the select condition. The
/// parameter may be used to differentiate a select with FP compare from
/// integer compare.
virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
return true;
}
/// Return true if multiple condition registers are available.
bool hasMultipleConditionRegisters() const {
return HasMultipleConditionRegisters;
}
/// Return true if the target has BitExtract instructions.
bool hasExtractBitsInsn() const { return HasExtractBitsInsn; }
/// Return the preferred vector type legalization action.
virtual TargetLoweringBase::LegalizeTypeAction
getPreferredVectorAction(MVT VT) const {
// The default action for one element vectors is to scalarize
if (VT.getVectorElementCount().isScalar())
return TypeScalarizeVector;
// The default action for an odd-width vector is to widen.
if (!VT.isPow2VectorType())
return TypeWidenVector;
// The default action for other vectors is to promote
return TypePromoteInteger;
}
// Return true if the half type should be passed around as i16, but promoted
// to float around arithmetic. The default behavior is to pass around as
// float and convert around loads/stores/bitcasts and other places where
// the size matters.
virtual bool softPromoteHalfType() const { return false; }
// There are two general methods for expanding a BUILD_VECTOR node:
// 1. Use SCALAR_TO_VECTOR on the defined scalar values and then shuffle
// them together.
// 2. Build the vector on the stack and then load it.
// If this function returns true, then method (1) will be used, subject to
// the constraint that all of the necessary shuffles are legal (as determined
// by isShuffleMaskLegal). If this function returns false, then method (2) is
// always used. The vector type, and the number of defined values, are
// provided.
virtual bool
shouldExpandBuildVectorWithShuffles(EVT /* VT */,
unsigned DefinedValues) const {
return DefinedValues < 3;
}
/// Return true if integer divide is usually cheaper than a sequence of
/// several shifts, adds, and multiplies for this target.
/// The definition of "cheaper" may depend on whether we're optimizing
/// for speed or for size.
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const { return false; }
/// Return true if the target can handle a standalone remainder operation.
virtual bool hasStandaloneRem(EVT VT) const {
return true;
}
/// Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const {
// Default behavior is to replace SQRT(X) with X*RSQRT(X).
return false;
}
/// Reciprocal estimate status values used by the functions below.
enum ReciprocalEstimate : int {
Unspecified = -1,
Disabled = 0,
Enabled = 1
};
/// Return a ReciprocalEstimate enum value for a square root of the given type
/// based on the function's attributes. If the operation is not overridden by
/// the function's attributes, "Unspecified" is returned and target defaults
/// are expected to be used for instruction selection.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const;
/// Return a ReciprocalEstimate enum value for a division of the given type
/// based on the function's attributes. If the operation is not overridden by
/// the function's attributes, "Unspecified" is returned and target defaults
/// are expected to be used for instruction selection.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const;
/// Return the refinement step count for a square root of the given type based
/// on the function's attributes. If the operation is not overridden by
/// the function's attributes, "Unspecified" is returned and target defaults
/// are expected to be used for instruction selection.
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const;
/// Return the refinement step count for a division of the given type based
/// on the function's attributes. If the operation is not overridden by
/// the function's attributes, "Unspecified" is returned and target defaults
/// are expected to be used for instruction selection.
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const;
/// Returns true if target has indicated at least one type should be bypassed.
bool isSlowDivBypassed() const { return !BypassSlowDivWidths.empty(); }
/// Returns map of slow types for division or remainder with corresponding
/// fast types
const DenseMap<unsigned int, unsigned int> &getBypassSlowDivWidths() const {
return BypassSlowDivWidths;
}
/// Return true if Flow Control is an expensive operation that should be
/// avoided.
bool isJumpExpensive() const { return JumpIsExpensive; }
/// Return true if selects are only cheaper than branches if the branch is
/// unlikely to be predicted right.
bool isPredictableSelectExpensive() const {
return PredictableSelectIsExpensive;
}
virtual bool fallBackToDAGISel(const Instruction &Inst) const {
return false;
}
/// If a branch or a select condition is skewed in one direction by more than
/// this factor, it is very likely to be predicted correctly.
virtual BranchProbability getPredictableBranchThreshold() const;
/// Return true if the following transform is beneficial:
/// fold (conv (load x)) -> (load (conv*)x)
/// On architectures that don't natively support some vector loads
/// efficiently, casting the load to a smaller vector of larger types and
/// loading is more efficient, however, this can be undone by optimizations in
/// dag combiner.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
const SelectionDAG &DAG,
const MachineMemOperand &MMO) const {
// Don't do if we could do an indexed load on the original type, but not on
// the new one.
if (!LoadVT.isSimple() || !BitcastVT.isSimple())
return true;
MVT LoadMVT = LoadVT.getSimpleVT();
// Don't bother doing this if it's just going to be promoted again later, as
// doing so might interfere with other combines.
if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&
getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
return false;
bool Fast = false;
return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
MMO, &Fast) && Fast;
}
/// Return true if the following transform is beneficial:
/// (store (y (conv x)), y*)) -> (store x, (x*))
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT,
const SelectionDAG &DAG,
const MachineMemOperand &MMO) const {
// Default to the same logic as loads.
return isLoadBitCastBeneficial(StoreVT, BitcastVT, DAG, MMO);
}
/// Return true if it is expected to be cheaper to do a store of a non-zero
/// vector constant with the given size and type for the address space than to
/// store the individual scalar element constants.
virtual bool storeOfVectorConstantIsCheap(EVT MemVT,
unsigned NumElem,
unsigned AddrSpace) const {
return false;
}
/// Allow store merging for the specified type after legalization in addition
/// to before legalization. This may transform stores that do not exist
/// earlier (for example, stores created from intrinsics).
virtual bool mergeStoresAfterLegalization(EVT MemVT) const {
return true;
}
/// Returns if it's reasonable to merge stores to MemVT size.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT,
const SelectionDAG &DAG) const {
return true;
}
/// Return true if it is cheap to speculate a call to intrinsic cttz.
virtual bool isCheapToSpeculateCttz() const {
return false;
}
/// Return true if it is cheap to speculate a call to intrinsic ctlz.
virtual bool isCheapToSpeculateCtlz() const {
return false;
}
/// Return true if ctlz instruction is fast.
virtual bool isCtlzFast() const {
return false;
}
/// Return the maximum number of "x & (x - 1)" operations that can be done
/// instead of deferring to a custom CTPOP.
virtual unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const {
return 1;
}
/// Return true if instruction generated for equality comparison is folded
/// with instruction generated for signed comparison.
virtual bool isEqualityCmpFoldedWithSignedCmp() const { return true; }
/// Return true if it is safe to transform an integer-domain bitwise operation
/// into the equivalent floating-point operation. This should be set to true
/// if the target has IEEE-754-compliant fabs/fneg operations for the input
/// type.
virtual bool hasBitPreservingFPLogic(EVT VT) const {
return false;
}
/// Return true if it is cheaper to split the store of a merged int val
/// from a pair of smaller values into multiple stores.
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const {
return false;
}
/// Return if the target supports combining a
/// chain like:
/// \code
/// %andResult = and %val1, #mask
/// %icmpResult = icmp %andResult, 0
/// \endcode
/// into a single machine instruction of a form like:
/// \code
/// cc = test %register, #mask
/// \endcode
virtual bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
return false;
}
/// Use bitwise logic to make pairs of compares more efficient. For example:
/// and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
/// This should be true when it takes more than one instruction to lower
/// setcc (cmp+set on x86 scalar), when bitwise ops are faster than logic on
/// condition bits (crand on PowerPC), and/or when reducing cmp+br is a win.
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const {
return false;
}
/// Return the preferred operand type if the target has a quick way to compare
/// integer values of the given size. Assume that any legal integer type can
/// be compared efficiently. Targets may override this to allow illegal wide
/// types to return a vector type if there is support to compare that type.
virtual MVT hasFastEqualityCompare(unsigned NumBits) const {
MVT VT = MVT::getIntegerVT(NumBits);
return isTypeLegal(VT) ? VT : MVT::INVALID_SIMPLE_VALUE_TYPE;
}
/// Return true if the target should transform:
/// (X & Y) == Y ---> (~X & Y) == 0
/// (X & Y) != Y ---> (~X & Y) != 0
///
/// This may be profitable if the target has a bitwise and-not operation that
/// sets comparison flags. A target may want to limit the transformation based
/// on the type of Y or if Y is a constant.
///
/// Note that the transform will not occur if Y is known to be a power-of-2
/// because a mask and compare of a single bit can be handled by inverting the
/// predicate, for example:
/// (X & 8) == 8 ---> (X & 8) != 0
virtual bool hasAndNotCompare(SDValue Y) const {
return false;
}
/// Return true if the target has a bitwise and-not operation:
/// X = ~A & B
/// This can be used to simplify select or other instructions.
virtual bool hasAndNot(SDValue X) const {
// If the target has the more complex version of this operation, assume that
// it has this operation too.
return hasAndNotCompare(X);
}
/// Return true if the target has a bit-test instruction:
/// (X & (1 << Y)) ==/!= 0
/// This knowledge can be used to prevent breaking the pattern,
/// or creating it if it could be recognized.
virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; }
/// There are two ways to clear extreme bits (either low or high):
/// Mask: x & (-1 << y) (the instcombine canonical form)
/// Shifts: x >> y << y
/// Return true if the variant with 2 variable shifts is preferred.
/// Return false if there is no preference.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const {
// By default, let's assume that no one prefers shifts.
return false;
}
/// Return true if it is profitable to fold a pair of shifts into a mask.
/// This is usually true on most targets. But some targets, like Thumb1,
/// have immediate shift instructions, but no immediate "and" instruction;
/// this makes the fold unprofitable.
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N,
CombineLevel Level) const {
return true;
}
/// Should we tranform the IR-optimal check for whether given truncation
/// down into KeptBits would be truncating or not:
/// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
/// Into it's more traditional form:
/// ((%x << C) a>> C) dstcond %x
/// Return true if we should transform.
/// Return false if there is no preference.
virtual bool shouldTransformSignedTruncationCheck(EVT XVT,
unsigned KeptBits) const {
// By default, let's assume that no one prefers shifts.
return false;
}
/// Given the pattern
/// (X & (C l>>/<< Y)) ==/!= 0
/// return true if it should be transformed into:
/// ((X <</l>> Y) & C) ==/!= 0
/// WARNING: if 'X' is a constant, the fold may deadlock!
/// FIXME: we could avoid passing XC, but we can't use isConstOrConstSplat()
/// here because it can end up being not linked in.
virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
unsigned OldShiftOpcode, unsigned NewShiftOpcode,
SelectionDAG &DAG) const {
if (hasBitTest(X, Y)) {
// One interesting pattern that we'd want to form is 'bit test':
// ((1 << Y) & C) ==/!= 0
// But we also need to be careful not to try to reverse that fold.
// Is this '1 << Y' ?
if (OldShiftOpcode == ISD::SHL && CC->isOne())
return false; // Keep the 'bit test' pattern.
// Will it be '1 << Y' after the transform ?
if (XC && NewShiftOpcode == ISD::SHL && XC->isOne())
return true; // Do form the 'bit test' pattern.
}
// If 'X' is a constant, and we transform, then we will immediately
// try to undo the fold, thus causing endless combine loop.
// So by default, let's assume everyone prefers the fold
// iff 'X' is not a constant.
return !XC;
}
/// These two forms are equivalent:
/// sub %y, (xor %x, -1)
/// add (add %x, 1), %y
/// The variant with two add's is IR-canonical.
/// Some targets may prefer one to the other.
virtual bool preferIncOfAddToSubOfNot(EVT VT) const {
// By default, let's assume that everyone prefers the form with two add's.
return true;
}
/// Return true if the target wants to use the optimization that
/// turns ext(promotableInst1(...(promotableInstN(load)))) into
/// promotedInst1(...(promotedInstN(ext(load)))).
bool enableExtLdPromotion() const { return EnableExtLdPromotion; }
/// Return true if the target can combine store(extractelement VectorTy,
/// Idx).
/// \p Cost[out] gives the cost of that transformation when this is true.
virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
unsigned &Cost) const {
return false;
}
/// Return true if inserting a scalar into a variable element of an undef
/// vector is more efficiently handled by splatting the scalar instead.
virtual bool shouldSplatInsEltVarIndex(EVT) const {
return false;
}
/// Return true if target always beneficiates from combining into FMA for a
/// given value type. This must typically return false on targets where FMA
/// takes more cycles to execute than FADD.
virtual bool enableAggressiveFMAFusion(EVT VT) const {
return false;
}
/// Return the ValueType of the result of SETCC operations.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
EVT VT) const;
/// Return the ValueType for comparison libcalls. Comparions libcalls include
/// floating point comparion calls, and Ordered/Unordered check calls on
/// floating point numbers.
virtual
MVT::SimpleValueType getCmpLibcallReturnType() const;
/// For targets without i1 registers, this gives the nature of the high-bits
/// of boolean values held in types wider than i1.
///
/// "Boolean values" are special true/false values produced by nodes like
/// SETCC and consumed (as the condition) by nodes like SELECT and BRCOND.
/// Not to be confused with general values promoted from i1. Some cpus
/// distinguish between vectors of boolean and scalars; the isVec parameter
/// selects between the two kinds. For example on X86 a scalar boolean should
/// be zero extended from i1, while the elements of a vector of booleans
/// should be sign extended from i1.
///
/// Some cpus also treat floating point types the same way as they treat
/// vectors instead of the way they treat scalars.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const {
if (isVec)
return BooleanVectorContents;
return isFloat ? BooleanFloatContents : BooleanContents;
}
BooleanContent getBooleanContents(EVT Type) const {
return getBooleanContents(Type.isVector(), Type.isFloatingPoint());
}
/// Return target scheduling preference.
Sched::Preference getSchedulingPreference() const {
return SchedPreferenceInfo;
}
/// Some scheduler, e.g. hybrid, can switch to different scheduling heuristics
/// for different nodes. This function returns the preference (or none) for
/// the given node.
virtual Sched::Preference getSchedulingPreference(SDNode *) const {
return Sched::None;
}
/// Return the register class that should be used for the specified value
/// type.
virtual const TargetRegisterClass *getRegClassFor(MVT VT, bool isDivergent = false) const {
(void)isDivergent;
const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
assert(RC && "This value type is not natively supported!");
return RC;
}
/// Allows target to decide about the register class of the
/// specific value that is live outside the defining block.
/// Returns true if the value needs uniform register class.
virtual bool requiresUniformRegister(MachineFunction &MF,
const Value *) const {
return false;
}
/// Return the 'representative' register class for the specified value
/// type.
///
/// The 'representative' register class is the largest legal super-reg
/// register class for the register class of the value type. For example, on
/// i386 the rep register class for i8, i16, and i32 are GR32; while the rep
/// register class is GR64 on x86_64.
virtual const TargetRegisterClass *getRepRegClassFor(MVT VT) const {
const TargetRegisterClass *RC = RepRegClassForVT[VT.SimpleTy];
return RC;
}
/// Return the cost of the 'representative' register class for the specified
/// value type.
virtual uint8_t getRepRegClassCostFor(MVT VT) const {
return RepRegClassCostForVT[VT.SimpleTy];
}
/// Return true if SHIFT instructions should be expanded to SHIFT_PARTS
/// instructions, and false if a library call is preferred (e.g for code-size
/// reasons).
virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const {
return true;
}
/// Return true if the target has native support for the specified value type.
/// This means that it has a register that directly holds it without
/// promotions or expansions.
bool isTypeLegal(EVT VT) const {
assert(!VT.isSimple() ||
(unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegClassForVT));
return VT.isSimple() && RegClassForVT[VT.getSimpleVT().SimpleTy] != nullptr;
}
class ValueTypeActionImpl {
/// ValueTypeActions - For each value type, keep a LegalizeTypeAction enum
/// that indicates how instruction selection should deal with the type.
LegalizeTypeAction ValueTypeActions[MVT::LAST_VALUETYPE];
public:
ValueTypeActionImpl() {
std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions),
TypeLegal);
}
LegalizeTypeAction getTypeAction(MVT VT) const {
return ValueTypeActions[VT.SimpleTy];
}
void setTypeAction(MVT VT, LegalizeTypeAction Action) {
ValueTypeActions[VT.SimpleTy] = Action;
}
};
const ValueTypeActionImpl &getValueTypeActions() const {
return ValueTypeActions;
}
/// Return how we should legalize values of this type, either it is already
/// legal (return 'Legal') or we need to promote it to a larger type (return
/// 'Promote'), or we need to expand it into multiple registers of smaller
/// integer type (return 'Expand'). 'Custom' is not an option.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const {
return getTypeConversion(Context, VT).first;
}
LegalizeTypeAction getTypeAction(MVT VT) const {
return ValueTypeActions.getTypeAction(VT);
}
/// For types supported by the target, this is an identity function. For
/// types that must be promoted to larger types, this returns the larger type
/// to promote to. For integer types that are larger than the largest integer
/// register, this contains one step in the expansion to get to the smaller
/// register. For illegal floating point types, this returns the integer type
/// to transform to.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const {
return getTypeConversion(Context, VT).second;
}
/// For types supported by the target, this is an identity function. For
/// types that must be expanded (i.e. integer types that are larger than the
/// largest integer register or illegal floating point types), this returns
/// the largest legal type it will be expanded to.
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const {
assert(!VT.isVector());
while (true) {
switch (getTypeAction(Context, VT)) {
case TypeLegal:
return VT;
case TypeExpandInteger:
VT = getTypeToTransformTo(Context, VT);
break;
default:
llvm_unreachable("Type is not legal nor is it to be expanded!");
}
}
}
/// Vector types are broken down into some number of legal first class types.
/// For example, EVT::v8f32 maps to 2 EVT::v4f32 with Altivec or SSE1, or 8
/// promoted EVT::f64 values with the X86 FP stack. Similarly, EVT::v2i64
/// turns into 4 EVT::i32 values with both PPC and X86.
///
/// This method returns the number of registers needed, and the VT for each
/// register. It also returns the VT and quantity of the intermediate values
/// before they are promoted/expanded.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
EVT &IntermediateVT,
unsigned &NumIntermediates,
MVT &RegisterVT) const;
/// Certain targets such as MIPS require that some types such as vectors are
/// always broken down into scalars in some contexts. This occurs even if the
/// vector type is legal.
virtual unsigned getVectorTypeBreakdownForCallingConv(
LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
unsigned &NumIntermediates, MVT &RegisterVT) const {
return getVectorTypeBreakdown(Context, VT, IntermediateVT, NumIntermediates,
RegisterVT);
}
struct IntrinsicInfo {
unsigned opc = 0; // target opcode
EVT memVT; // memory VT
// value representing memory location
PointerUnion<const Value *, const PseudoSourceValue *> ptrVal;
int offset = 0; // offset off of ptrVal
uint64_t size = 0; // the size of the memory location
// (taken from memVT if zero)
MaybeAlign align = Align(1); // alignment
MachineMemOperand::Flags flags = MachineMemOperand::MONone;
IntrinsicInfo() = default;
};
/// Given an intrinsic, checks if on the target the intrinsic will need to map
/// to a MemIntrinsicNode (touches memory). If this is the case, it returns
/// true and store the intrinsic information into the IntrinsicInfo that was
/// passed to the function.
virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
MachineFunction &,
unsigned /*Intrinsic*/) const {