diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 67141f3a63260..e804c9d26ccc7 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -402,6 +402,7 @@ class LegalizerHelper { LegalizeResult lowerDIVREM(MachineInstr &MI); LegalizeResult lowerAbsToAddXor(MachineInstr &MI); LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI); + LegalizeResult lowerIsNaN(MachineInstr &MI); }; /// Helper function that creates a libcall to the given \p Name using the given diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 2ce32748453d2..611cf1055a1b1 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -3486,6 +3487,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { case G_ROTL: case G_ROTR: return lowerRotate(MI); + case G_ISNAN: + return lowerIsNaN(MI); } } @@ -7355,3 +7358,34 @@ LegalizerHelper::lowerAbsToMaxNeg(MachineInstr &MI) { MI.eraseFromParent(); return Legalized; } + +LegalizerHelper::LegalizeResult LegalizerHelper::lowerIsNaN(MachineInstr &MI) { + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + LLT SrcTy = MRI.getType(Src); + if (MI.getFlags() & MachineInstr::NoFPExcept) { + // Lower to an unordered comparison. + auto Zero = MIRBuilder.buildFConstant(SrcTy, 0.0); + MIRBuilder.buildFCmp(CmpInst::Predicate::FCMP_UNO, Dst, Src, Zero); + MI.eraseFromParent(); + return Legalized; + } + + // Use integer operations to avoid traps if the argument is SNaN. + + // NaN has all exp bits set and a non zero significand. Therefore: + // isnan(V) == exp mask < abs(V) + auto FPToSI = MIRBuilder.buildFPTOSI(SrcTy, Src); + auto Mask = APInt::getSignedMaxValue(SrcTy.getScalarSizeInBits()); + auto MaskCst = MIRBuilder.buildConstant(SrcTy, Mask); + auto AbsV = MIRBuilder.buildAnd(SrcTy, FPToSI, MaskCst); + auto *FloatTy = getFloatTypeForLLT(MI.getMF()->getFunction().getContext(), + SrcTy.getScalarType()); + if (!FloatTy) + return UnableToLegalize; + auto ExpMask = APFloat::getInf(FloatTy->getFltSemantics()).bitcastToAPInt(); + auto ExpMaskCst = MIRBuilder.buildConstant(SrcTy, ExpMask); + MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_SLT, Dst, ExpMaskCst, AbsV); + MI.eraseFromParent(); + return Legalized; +} diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 26af9e2c2b809..b28eef9d2223a 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -727,6 +727,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) // TODO: Vector types. getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT}).lowerIf(isScalar(0)); + getActionDefinitionsBuilder(G_ISNAN).lower(); + getLegacyLegalizerInfo().computeTables(); verify(*ST.getInstrInfo()); } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-isnan.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-isnan.mir new file mode 100644 index 0000000000000..4a9be4cedf3aa --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-isnan.mir @@ -0,0 +1,112 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=legalizer -global-isel-abort=0 -verify-machineinstrs %s -o - | FileCheck %s + +... +--- +name: scalar_nofpexcept +tracksRegLiveness: true +body: | + bb.0: + liveins: $h0 + + ; CHECK-LABEL: name: scalar_nofpexcept + ; CHECK: liveins: $h0 + ; CHECK: %val:_(s16) = COPY $h0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00 + ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[C]](s32) + ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %val(s16) + ; CHECK: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16) + ; CHECK: [[FCMP:%[0-9]+]]:_(s32) = G_FCMP floatpred(uno), [[FPEXT]](s32), [[FPEXT1]] + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[FCMP]](s32) + ; CHECK: %ext:_(s32) = G_AND [[COPY]], [[C1]] + ; CHECK: $w0 = COPY %ext(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val:_(s16) = COPY $h0 + %isnan:_(s1) = nofpexcept G_ISNAN %val(s16) + %ext:_(s32) = G_ZEXT %isnan(s1) + $w0 = COPY %ext(s32) + RET_ReallyLR implicit $w0 + +... +--- +name: vector_nofpexcept +tracksRegLiveness: true +body: | + bb.0: + liveins: $d0 + + ; CHECK-LABEL: name: vector_nofpexcept + ; CHECK: liveins: $d0 + ; CHECK: %val:_(<4 x s16>) = COPY $d0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00 + ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[C]](s32) + ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC]](s16), [[FPTRUNC]](s16), [[FPTRUNC]](s16) + ; CHECK: [[FCMP:%[0-9]+]]:_(<4 x s16>) = G_FCMP floatpred(uno), %val(<4 x s16>), [[BUILD_VECTOR]] + ; CHECK: %ext:_(<4 x s16>) = COPY [[FCMP]](<4 x s16>) + ; CHECK: $d0 = COPY %ext(<4 x s16>) + ; CHECK: RET_ReallyLR implicit $d0 + %val:_(<4 x s16>) = COPY $d0 + %isnan:_(<4 x s1>) = nofpexcept G_ISNAN %val(<4 x s16>) + %ext:_(<4 x s16>) = G_ANYEXT %isnan(<4 x s1>) + $d0 = COPY %ext(<4 x s16>) + RET_ReallyLR implicit $d0 + +... +--- +name: scalar_no_flags +tracksRegLiveness: true +body: | + bb.0: + liveins: $h0 + + ; CHECK-LABEL: name: scalar_no_flags + ; CHECK: liveins: $h0 + ; CHECK: %val:_(s16) = COPY $h0 + ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT %val(s16) + ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FPEXT]](s32) + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[FPTOSI]](s32) + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767 + ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]] + ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31744 + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[AND]](s32) + ; CHECK: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY1]], 16 + ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[C1]](s32), [[SEXT_INREG]] + ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32) + ; CHECK: %ext:_(s32) = G_AND [[COPY2]], [[C2]] + ; CHECK: $w0 = COPY %ext(s32) + ; CHECK: RET_ReallyLR implicit $w0 + %val:_(s16) = COPY $h0 + %isnan:_(s1) = G_ISNAN %val(s16) + %ext:_(s32) = G_ZEXT %isnan(s1) + $w0 = COPY %ext(s32) + RET_ReallyLR implicit $w0 + +... +--- +name: vector_no_flags +tracksRegLiveness: true +body: | + bb.0: + liveins: $d0 + + ; CHECK-LABEL: name: vector_no_flags + ; CHECK: liveins: $d0 + ; CHECK: %val:_(<4 x s16>) = COPY $d0 + ; CHECK: [[FPTOSI:%[0-9]+]]:_(<4 x s16>) = G_FPTOSI %val(<4 x s16>) + ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 32767 + ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16) + ; CHECK: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[FPTOSI]], [[BUILD_VECTOR]] + ; CHECK: [[C1:%[0-9]+]]:_(s16) = G_CONSTANT i16 31744 + ; CHECK: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[C1]](s16), [[C1]](s16), [[C1]](s16), [[C1]](s16) + ; CHECK: [[ICMP:%[0-9]+]]:_(<4 x s16>) = G_ICMP intpred(slt), [[BUILD_VECTOR1]](<4 x s16>), [[AND]] + ; CHECK: %isnan:_(<4 x s1>) = G_TRUNC [[ICMP]](<4 x s16>) + ; CHECK: %ext:_(<4 x s16>) = G_ANYEXT %isnan(<4 x s1>) + ; CHECK: $d0 = COPY %ext(<4 x s16>) + ; CHECK: RET_ReallyLR implicit $d0 + %val:_(<4 x s16>) = COPY $d0 + %isnan:_(<4 x s1>) = G_ISNAN %val(<4 x s16>) + %ext:_(<4 x s16>) = G_ANYEXT %isnan(<4 x s1>) + $d0 = COPY %ext(<4 x s16>) + RET_ReallyLR implicit $d0