Skip to content

Commit 9c0743f

Browse files
jyli0116jyli0116
authored
[GlobalISel] Allow expansion of urem by constant in prelegalizer (#145914)
This patch allows urem by a constant to be expanded more efficiently to avoid the need for expensive udiv instructions. This is part of the resolution to issue #118090
1 parent 0aafeb8 commit 9c0743f

File tree

7 files changed

+3779
-607
lines changed

7 files changed

+3779
-607
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -693,18 +693,19 @@ class CombinerHelper {
693693
/// feeding a G_AND instruction \p MI.
694694
bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo) const;
695695

696-
/// Given an G_UDIV \p MI expressing a divide by constant, return an
697-
/// expression that implements it by multiplying by a magic number.
696+
/// Given an G_UDIV \p MI or G_UREM \p MI expressing a divide by constant,
697+
/// return an expression that implements it by multiplying by a magic number.
698698
/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
699-
MachineInstr *buildUDivUsingMul(MachineInstr &MI) const;
700-
/// Combine G_UDIV by constant into a multiply by magic constant.
701-
bool matchUDivByConst(MachineInstr &MI) const;
702-
void applyUDivByConst(MachineInstr &MI) const;
699+
MachineInstr *buildUDivorURemUsingMul(MachineInstr &MI) const;
700+
/// Combine G_UDIV or G_UREM by constant into a multiply by magic constant.
701+
bool matchUDivorURemByConst(MachineInstr &MI) const;
702+
void applyUDivorURemByConst(MachineInstr &MI) const;
703703

704704
/// Given an G_SDIV \p MI expressing a signed divide by constant, return an
705705
/// expression that implements it by multiplying by a magic number.
706706
/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
707707
MachineInstr *buildSDivUsingMul(MachineInstr &MI) const;
708+
/// Combine G_SDIV by constant into a multiply by magic constant.
708709
bool matchSDivByConst(MachineInstr &MI) const;
709710
void applySDivByConst(MachineInstr &MI) const;
710711

‎llvm/include/llvm/Target/GlobalISel/Combine.td

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,8 +1132,8 @@ def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg,
11321132
def udiv_by_const : GICombineRule<
11331133
(defs root:$root),
11341134
(match (wip_match_opcode G_UDIV):$root,
1135-
[{ return Helper.matchUDivByConst(*${root}); }]),
1136-
(apply [{ Helper.applyUDivByConst(*${root}); }])>;
1135+
[{ return Helper.matchUDivorURemByConst(*${root}); }]),
1136+
(apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
11371137

11381138
def sdiv_by_const : GICombineRule<
11391139
(defs root:$root),
@@ -1156,6 +1156,14 @@ def udiv_by_pow2 : GICombineRule<
11561156
def intdiv_combines : GICombineGroup<[udiv_by_const, sdiv_by_const,
11571157
sdiv_by_pow2, udiv_by_pow2]>;
11581158

1159+
def urem_by_const : GICombineRule<
1160+
(defs root:$root),
1161+
(match (G_UREM $dst, $x, $y):$root,
1162+
[{ return Helper.matchUDivorURemByConst(*${root}); }]),
1163+
(apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
1164+
1165+
def intrem_combines : GICombineGroup<[urem_by_const]>;
1166+
11591167
def reassoc_ptradd : GICombineRule<
11601168
(defs root:$root, build_fn_matchinfo:$matchinfo),
11611169
(match (wip_match_opcode G_PTR_ADD):$root,
@@ -2048,7 +2056,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
20482056
constant_fold_cast_op, fabs_fneg_fold,
20492057
intdiv_combines, mulh_combines, redundant_neg_operands,
20502058
and_or_disjoint_mask, fma_combines, fold_binop_into_select,
2051-
sub_add_reg, select_to_minmax,
2059+
intrem_combines, sub_add_reg, select_to_minmax,
20522060
fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors,
20532061
simplify_neg_minmax, combine_concat_vector,
20542062
sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,

‎llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5295,12 +5295,13 @@ bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
52955295
return false;
52965296
}
52975297

5298-
MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) const {
5299-
assert(MI.getOpcode() == TargetOpcode::G_UDIV);
5300-
auto &UDiv = cast<GenericMachineInstr>(MI);
5301-
Register Dst = UDiv.getReg(0);
5302-
Register LHS = UDiv.getReg(1);
5303-
Register RHS = UDiv.getReg(2);
5298+
MachineInstr *CombinerHelper::buildUDivorURemUsingMul(MachineInstr &MI) const {
5299+
unsigned Opcode = MI.getOpcode();
5300+
assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
5301+
auto &UDivorRem = cast<GenericMachineInstr>(MI);
5302+
Register Dst = UDivorRem.getReg(0);
5303+
Register LHS = UDivorRem.getReg(1);
5304+
Register RHS = UDivorRem.getReg(2);
53045305
LLT Ty = MRI.getType(Dst);
53055306
LLT ScalarTy = Ty.getScalarType();
53065307
const unsigned EltBits = ScalarTy.getScalarSizeInBits();
@@ -5453,11 +5454,18 @@ MachineInstr *CombinerHelper::buildUDivUsingMul(MachineInstr &MI) const {
54535454
auto IsOne = MIB.buildICmp(
54545455
CmpInst::Predicate::ICMP_EQ,
54555456
Ty.isScalar() ? LLT::scalar(1) : Ty.changeElementSize(1), RHS, One);
5456-
return MIB.buildSelect(Ty, IsOne, LHS, Q);
5457+
auto ret = MIB.buildSelect(Ty, IsOne, LHS, Q);
5458+
5459+
if (Opcode == TargetOpcode::G_UREM) {
5460+
auto Prod = MIB.buildMul(Ty, ret, RHS);
5461+
return MIB.buildSub(Ty, LHS, Prod);
5462+
}
5463+
return ret;
54575464
}
54585465

5459-
bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
5460-
assert(MI.getOpcode() == TargetOpcode::G_UDIV);
5466+
bool CombinerHelper::matchUDivorURemByConst(MachineInstr &MI) const {
5467+
unsigned Opcode = MI.getOpcode();
5468+
assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
54615469
Register Dst = MI.getOperand(0).getReg();
54625470
Register RHS = MI.getOperand(2).getReg();
54635471
LLT DstTy = MRI.getType(Dst);
@@ -5474,7 +5482,8 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
54745482
if (MF.getFunction().hasMinSize())
54755483
return false;
54765484

5477-
if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
5485+
if (Opcode == TargetOpcode::G_UDIV &&
5486+
MI.getFlag(MachineInstr::MIFlag::IsExact)) {
54785487
return matchUnaryPredicate(
54795488
MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
54805489
}
@@ -5494,14 +5503,17 @@ bool CombinerHelper::matchUDivByConst(MachineInstr &MI) const {
54945503
{DstTy.isVector() ? DstTy.changeElementSize(1) : LLT::scalar(1),
54955504
DstTy}}))
54965505
return false;
5506+
if (Opcode == TargetOpcode::G_UREM &&
5507+
!isLegalOrBeforeLegalizer({TargetOpcode::G_SUB, {DstTy, DstTy}}))
5508+
return false;
54975509
}
54985510

54995511
return matchUnaryPredicate(
55005512
MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
55015513
}
55025514

5503-
void CombinerHelper::applyUDivByConst(MachineInstr &MI) const {
5504-
auto *NewMI = buildUDivUsingMul(MI);
5515+
void CombinerHelper::applyUDivorURemByConst(MachineInstr &MI) const {
5516+
auto *NewMI = buildUDivorURemUsingMul(MI);
55055517
replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg());
55065518
}
55075519

‎llvm/test/CodeGen/AArch64/pr58431.ll

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,12 @@
44
define i32 @f(i64 %0) {
55
; CHECK-LABEL: f:
66
; CHECK: // %bb.0:
7-
; CHECK-NEXT: mov w8, #10 // =0xa
7+
; CHECK-NEXT: mov x8, #-7378697629483820647 // =0x9999999999999999
88
; CHECK-NEXT: mov w9, w0
9-
; CHECK-NEXT: udiv x10, x9, x8
10-
; CHECK-NEXT: msub x0, x10, x8, x9
9+
; CHECK-NEXT: mov w10, #10 // =0xa
10+
; CHECK-NEXT: eor x8, x8, #0x8000000000000003
11+
; CHECK-NEXT: umulh x8, x9, x8
12+
; CHECK-NEXT: msub x0, x8, x10, x9
1113
; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
1214
; CHECK-NEXT: ret
1315
%2 = trunc i64 %0 to i32

0 commit comments

Comments
 (0)