[GlobalISel] Add support for *_fpmode intrinsics

The change implements support of the intrinsics `get_fpmode`,
`set_fpmode` and `reset_fpmode` in Global Instruction Selector. Now they
are lowered into library function calls.

Differential Revision: https://reviews.llvm.org/D158260

llvm/include/llvm/CodeGen/GlobalISel/Legalizer.h

@@ -66,8 +66,9 @@ class Legalizer : public MachineFunctionPass {
   }
 
   MachineFunctionProperties getClearedProperties() const override {
-    return MachineFunctionProperties().set(
-        MachineFunctionProperties::Property::NoPHIs);
+    return MachineFunctionProperties()
+        .set(MachineFunctionProperties::Property::NoPHIs)
+        .set(MachineFunctionProperties::Property::NoVRegs);
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override;

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

@@ -286,6 +286,14 @@ class LegalizerHelper {
                               uint64_t KnownLen, Align DstAlign, Align SrcAlign,
                               bool IsVolatile);
 
+  // Implements floating-point environment read/write via library function call.
+  LegalizeResult createGetStateLibcall(MachineIRBuilder &MIRBuilder,
+                                       MachineInstr &MI);
+  LegalizeResult createSetStateLibcall(MachineIRBuilder &MIRBuilder,
+                                       MachineInstr &MI);
+  LegalizeResult createResetStateLibcall(MachineIRBuilder &MIRBuilder,
+                                         MachineInstr &MI);
+
 public:
   /// Return the alignment to use for a stack temporary object with the given
   /// type.
@@ -441,6 +449,7 @@ LegalizerHelper::LegalizeResult
 createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                  MachineInstr &MI, LostDebugLocObserver &LocObserver);
 
+
 } // End namespace llvm.
 
 #endif

llvm/include/llvm/Support/TargetOpcodes.def

@@ -683,6 +683,11 @@ HANDLE_TARGET_OPCODE(G_FMAXNUM_IEEE)
 HANDLE_TARGET_OPCODE(G_FMINIMUM)
 HANDLE_TARGET_OPCODE(G_FMAXIMUM)
 
+/// Access to FP environment.
+HANDLE_TARGET_OPCODE(G_GET_FPMODE)
+HANDLE_TARGET_OPCODE(G_SET_FPMODE)
+HANDLE_TARGET_OPCODE(G_RESET_FPMODE)
+
 /// Generic pointer offset
 HANDLE_TARGET_OPCODE(G_PTR_ADD)
 

llvm/include/llvm/Target/GenericOpcodes.td

@@ -1008,6 +1008,35 @@ def G_FNEARBYINT : GenericInstruction {
   let hasSideEffects = false;
 }
 
+//------------------------------------------------------------------------------
+// Access to floating-point environment.
+//------------------------------------------------------------------------------
+
+// These operations read/write floating-point environment. The interaction with
+// it is modeled as a side effect, because constrained intrinsics use the same
+// method.
+
+// Reading floating-point control modes.
+def G_GET_FPMODE : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins);
+  let hasSideEffects = true;
+}
+
+// Setting floating-point control modes.
+def G_SET_FPMODE : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = true;
+}
+
+// Setting floating-point control modes to default state.
+def G_RESET_FPMODE : GenericInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins);
+  let hasSideEffects = true;
+}
+
 //------------------------------------------------------------------------------
 // Opcodes for LLVM Intrinsics
 //------------------------------------------------------------------------------

llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td

@@ -116,6 +116,10 @@ def : GINodeEquiv<G_INTRINSIC, intrinsic_wo_chain> {
   let IfConvergent = G_INTRINSIC_CONVERGENT;
 }
 
+def : GINodeEquiv<G_GET_FPMODE, get_fpmode>;
+def : GINodeEquiv<G_SET_FPMODE, set_fpmode>;
+def : GINodeEquiv<G_RESET_FPMODE, reset_fpmode>;
+
 // ISD::INTRINSIC_VOID can also be handled with G_INTRINSIC_W_SIDE_EFFECTS.
 let IfConvergent = G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS in {
   def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_void>;

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

@@ -1837,6 +1837,8 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
       return TargetOpcode::G_LROUND;
     case Intrinsic::llround:
       return TargetOpcode::G_LLROUND;
+    case Intrinsic::get_fpmode:
+      return TargetOpcode::G_GET_FPMODE;
   }
   return Intrinsic::not_intrinsic;
 }
@@ -2416,6 +2418,16 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
 
     return true;
   }
+  case Intrinsic::set_fpmode: {
+    Value *FPState = CI.getOperand(0);
+    MIRBuilder.buildInstr(TargetOpcode::G_SET_FPMODE, {},
+                          { getOrCreateVReg(*FPState) });
+    return true;
+  }
+  case Intrinsic::reset_fpmode: {
+    MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
+    return true;
+  }
 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)  \
   case Intrinsic::INTRINSIC:
 #include "llvm/IR/ConstrainedOps.def"

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

@@ -796,10 +796,134 @@ conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
                        {{MI.getOperand(1).getReg(), FromType, 0}});
 }
 
+static RTLIB::Libcall
+getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI) {
+  RTLIB::Libcall RTLibcall;
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_GET_FPMODE:
+    RTLibcall = RTLIB::FEGETMODE;
+    break;
+  case TargetOpcode::G_SET_FPMODE:
+  case TargetOpcode::G_RESET_FPMODE:
+    RTLibcall = RTLIB::FESETMODE;
+    break;
+  default:
+    llvm_unreachable("Unexpected opcode");
+  }
+  return RTLibcall;
+}
+
+// Some library functions that read FP state (fegetmode, fegetenv) write the
+// state into a region in memory. IR intrinsics that do the same operations
+// (get_fpmode, get_fpenv) return the state as integer value. To implement these
+// intrinsics via the library functions, we need to use temporary variable,
+// for example:
+//
+//     %0:_(s32) = G_GET_FPMODE
+//
+// is transformed to:
+//
+//     %1:_(p0) = G_FRAME_INDEX %stack.0
+//     BL &fegetmode
+//     %0:_(s32) = G_LOAD % 1
+//
+LegalizerHelper::LegalizeResult
+LegalizerHelper::createGetStateLibcall(MachineIRBuilder &MIRBuilder,
+                                       MachineInstr &MI) {
+  const DataLayout &DL = MIRBuilder.getDataLayout();
+  auto &MF = MIRBuilder.getMF();
+  auto &MRI = *MIRBuilder.getMRI();
+  auto &Ctx = MF.getFunction().getContext();
+
+  // Create temporary, where library function will put the read state.
+  Register Dst = MI.getOperand(0).getReg();
+  LLT StateTy = MRI.getType(Dst);
+  TypeSize StateSize = StateTy.getSizeInBytes();
+  Align TempAlign = getStackTemporaryAlignment(StateTy);
+  MachinePointerInfo TempPtrInfo;
+  auto Temp = createStackTemporary(StateSize, TempAlign, TempPtrInfo);
+
+  // Create a call to library function, with the temporary as an argument.
+  unsigned TempAddrSpace = DL.getAllocaAddrSpace();
+  Type *StatePtrTy = PointerType::get(Ctx, TempAddrSpace);
+  RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
+  auto Res =
+      createLibcall(MIRBuilder, RTLibcall,
+                    CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
+                    CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}));
+  if (Res != LegalizerHelper::Legalized)
+    return Res;
+
+  // Create a load from the temporary.
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      TempPtrInfo, MachineMemOperand::MOLoad, StateTy, TempAlign);
+  MIRBuilder.buildLoadInstr(TargetOpcode::G_LOAD, Dst, Temp, *MMO);
+
+  return LegalizerHelper::Legalized;
+}
+
+// Similar to `createGetStateLibcall` the function calls a library function
+// using transient space in stack. In this case the library function reads
+// content of memory region.
+LegalizerHelper::LegalizeResult
+LegalizerHelper::createSetStateLibcall(MachineIRBuilder &MIRBuilder,
+                                       MachineInstr &MI) {
+  const DataLayout &DL = MIRBuilder.getDataLayout();
+  auto &MF = MIRBuilder.getMF();
+  auto &MRI = *MIRBuilder.getMRI();
+  auto &Ctx = MF.getFunction().getContext();
+
+  // Create temporary, where library function will get the new state.
+  Register Src = MI.getOperand(0).getReg();
+  LLT StateTy = MRI.getType(Src);
+  TypeSize StateSize = StateTy.getSizeInBytes();
+  Align TempAlign = getStackTemporaryAlignment(StateTy);
+  MachinePointerInfo TempPtrInfo;
+  auto Temp = createStackTemporary(StateSize, TempAlign, TempPtrInfo);
+
+  // Put the new state into the temporary.
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+      TempPtrInfo, MachineMemOperand::MOStore, StateTy, TempAlign);
+  MIRBuilder.buildStore(Src, Temp, *MMO);
+
+  // Create a call to library function, with the temporary as an argument.
+  unsigned TempAddrSpace = DL.getAllocaAddrSpace();
+  Type *StatePtrTy = PointerType::get(Ctx, TempAddrSpace);
+  RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
+  return createLibcall(MIRBuilder, RTLibcall,
+                       CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
+                       CallLowering::ArgInfo({Temp.getReg(0), StatePtrTy, 0}));
+}
+
+// The function is used to legalize operations that set default environment
+// state. In C library a call like `fesetmode(FE_DFL_MODE)` is used for that.
+// On most targets supported in glibc FE_DFL_MODE is defined as
+// `((const femode_t *) -1)`. Such assumption is used here. If for some target
+// it is not true, the target must provide custom lowering.
+LegalizerHelper::LegalizeResult
+LegalizerHelper::createResetStateLibcall(MachineIRBuilder &MIRBuilder,
+                                         MachineInstr &MI) {
+  const DataLayout &DL = MIRBuilder.getDataLayout();
+  auto &MF = MIRBuilder.getMF();
+  auto &Ctx = MF.getFunction().getContext();
+
+  // Create an argument for the library function.
+  unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
+  Type *StatePtrTy = PointerType::get(Ctx, AddrSpace);
+  unsigned PtrSize = DL.getPointerSizeInBits(AddrSpace);
+  LLT MemTy = LLT::pointer(AddrSpace, PtrSize);
+  auto DefValue = MIRBuilder.buildConstant(LLT::scalar(PtrSize), -1LL);
+  DstOp Dest(MRI.createGenericVirtualRegister(MemTy));
+  MIRBuilder.buildIntToPtr(Dest, DefValue);
+
+  RTLIB::Libcall RTLibcall = getStateLibraryFunctionFor(MI, TLI);
+  return createLibcall(MIRBuilder, RTLibcall,
+                       CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx), 0),
+                       CallLowering::ArgInfo({ Dest.getReg(), StatePtrTy, 0}));
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
-  LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
-  unsigned Size = LLTy.getSizeInBits();
   auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
 
   switch (MI.getOpcode()) {
@@ -811,6 +935,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
   case TargetOpcode::G_SREM:
   case TargetOpcode::G_UREM:
   case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
+    LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
+    unsigned Size = LLTy.getSizeInBits();
     Type *HLTy = IntegerType::get(Ctx, Size);
     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     if (Status != Legalized)
@@ -841,6 +967,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
   case TargetOpcode::G_FRINT:
   case TargetOpcode::G_FNEARBYINT:
   case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
+    LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
+    unsigned Size = LLTy.getSizeInBits();
     Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
     if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
       LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
@@ -903,6 +1031,24 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
     MI.eraseFromParent();
     return Result;
   }
+  case TargetOpcode::G_GET_FPMODE: {
+    LegalizeResult Result = createGetStateLibcall(MIRBuilder, MI);
+    if (Result != Legalized)
+      return Result;
+    break;
+  }
+  case TargetOpcode::G_SET_FPMODE: {
+    LegalizeResult Result = createSetStateLibcall(MIRBuilder, MI);
+    if (Result != Legalized)
+      return Result;
+    break;
+  }
+  case TargetOpcode::G_RESET_FPMODE: {
+    LegalizeResult Result = createResetStateLibcall(MIRBuilder, MI);
+    if (Result != Legalized)
+      return Result;
+    break;
+  }
   }
 
   MI.eraseFromParent();

llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp

@@ -102,6 +102,7 @@ static bool hasNoSimpleLoops(const LegalizeRule &Rule, const LegalityQuery &Q,
   case Lower:
   case MoreElements:
   case FewerElements:
+  case Libcall:
     break;
   default:
     return Q.Types[Mutation.first] != Mutation.second;
@@ -118,6 +119,10 @@ static bool mutationIsSane(const LegalizeRule &Rule,
   if (Rule.getAction() == Custom || Rule.getAction() == Legal)
     return true;
 
+  // Skip null mutation.
+  if (!Mutation.second.isValid())
+    return true;
+
   const unsigned TypeIdx = Mutation.first;
   const LLT OldTy = Q.Types[TypeIdx];
   const LLT NewTy = Mutation.second;

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

@@ -972,6 +972,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
 
   getActionDefinitionsBuilder(G_FMAD).lower();
 
+  // Access to floating-point environment.
+  getActionDefinitionsBuilder({G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
+      .libcall();
+
   getLegacyLegalizerInfo().computeTables();
   verify(*ST.getInstrInfo());
 }

llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-fpenv.ll

@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=aarch64-linux-gnu -global-isel -stop-after=irtranslator %s -o - | FileCheck %s
+
+declare i32 @llvm.get.fpmode.i32()
+declare void @llvm.set.fpmode.i32(i32 %fpmode)
+declare void @llvm.reset.fpmode()
+
+define i32 @func_get_fpmode() #0 {
+  ; CHECK-LABEL: name: func_get_fpmode
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   [[GET_FPMODE:%[0-9]+]]:_(s32) = G_GET_FPMODE
+  ; CHECK-NEXT:   $w0 = COPY [[GET_FPMODE]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $w0
+entry:
+  %fpmode = call i32 @llvm.get.fpmode.i32()
+  ret i32 %fpmode
+}
+
+define void @func_set_fpmode(i32 %fpmode) #0 {
+  ; CHECK-LABEL: name: func_set_fpmode
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK-NEXT:   G_SET_FPMODE [[COPY]](s32)
+  ; CHECK-NEXT:   RET_ReallyLR
+entry:
+  call void @llvm.set.fpmode.i32(i32 %fpmode)
+  ret void
+}
+
+
+define void @func_reset() #0 {
+  ; CHECK-LABEL: name: func_reset
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   G_RESET_FPMODE
+  ; CHECK-NEXT:   RET_ReallyLR
+entry:
+  call void @llvm.reset.fpmode()
+  ret void
+}
+
+attributes #0 = { nounwind "use-soft-float"="true" }