Fold bitcasts to rdregion/wrregion by changing region parameters

This transformation reorders rdregion->bitcast to bitcast->rdregion and
bitcast->wrregion to wrregion->bitcast, so they can be baled later.

Author: skachkov-intel

IGC/VectorCompiler/lib/GenXCodeGen/GenXLegalization.cpp

@@ -377,7 +377,7 @@ class GenXLegalization : public FunctionPass {
     Fixed4 = nullptr;
     TwiceWidth = nullptr;
   }
-  unsigned getExecSizeAllowedBits(Instruction *Inst);
+  unsigned adjustTwiceWidthOrFixed4(const Bale &B);
   bool checkIfLongLongSupportNeeded(Instruction *Inst) const;
   void verifyLSCFence(const Instruction *Inst);
   void verifyLSCAtomic(const Instruction *Inst);
@@ -575,84 +575,30 @@ bool GenXLegalization::runOnFunction(Function &F) {
   return true;
 }
 
-/***********************************************************************
- * getExecSizeAllowedBits : get bitmap of allowed execution sizes
- *
- * Enter:   Inst = main instruction of bale
- *
- * Return:  bit N set if execution size 1<<N is allowed
- *
- * Most instructions have a minimum width of 1. But some instructions,
- * such as dp4 and lrp, have a minimum width of 4, and legalization cannot
- * allow such an instruction to be split to a smaller width.
- *
- * This also sets up fields in GenXLegalization: Fixed4 is set to a use
- * that is a FIXED4 operand, and TwiceWidth is set to a use that is a
- * TWICEWIDTH operand.
- */
-unsigned GenXLegalization::getExecSizeAllowedBits(Instruction *Inst) {
-
-  switch (Inst->getOpcode()) {
-  default:
-    break;
-  case BinaryOperator::SDiv:
-  case BinaryOperator::UDiv:
-  case BinaryOperator::SRem:
-  case BinaryOperator::URem:
-    // If integer division IS supported.
-    //   Set maximum SIMD width to 16:
-    //      Recent HW does not support SIMD16/SIMD32 division, however,
-    //      finalizer splits such SIMD16 operations and we piggy-back
-    //      on this behavior.
-    // If integer division IS NOT supported.
-    //   The expectation is for GenXEmulate pass to replace such operations
-    //   with emulation routines (which has no restriction on SIMD width)
-    return ST->hasIntDivRem32() ? 0x1f : 0x3f;
-  }
-
-  unsigned ID = GenXIntrinsic::getAnyIntrinsicID(Inst);
-  switch (ID) {
-    case GenXIntrinsic::genx_ssmad:
-    case GenXIntrinsic::genx_sumad:
-    case GenXIntrinsic::genx_usmad:
-    case GenXIntrinsic::genx_uumad:
-    case GenXIntrinsic::genx_ssmad_sat:
-    case GenXIntrinsic::genx_sumad_sat:
-    case GenXIntrinsic::genx_usmad_sat:
-    case GenXIntrinsic::genx_uumad_sat:
-    case Intrinsic::fma:
-      // Do not emit simd32 mad for pre-ICLLP.
-      return ST->isICLLPplus() ? 0x3f : 0x1f;
-    default:
-      break;
-  }
-
-  if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
-    // We have a call instruction, so we can assume it is an intrinsic since
-    // otherwise processInst would not have got as far as calling us as
-    // a non-intrinsic call forces isSplittable() to be false.
-    auto CalledF = CI->getCalledFunction();
-    IGC_ASSERT(CalledF);
-    GenXIntrinsicInfo II(GenXIntrinsic::getAnyIntrinsicID(CalledF));
-    // While we have the intrinsic info, we also spot whether we have a FIXED4
-    // operand and/or a TWICEWIDTH operand.
+unsigned GenXLegalization::adjustTwiceWidthOrFixed4(const Bale &B) {
+  auto Main = B.getMainInst();
+  if (!Main)
+    return 0x3f;
+  // Spot whether we have a FIXED operand and/or a TWICEWIDTH operand.
+  if (GenXIntrinsic::isGenXIntrinsic(Main->Inst)) {
+    GenXIntrinsicInfo II(GenXIntrinsic::getAnyIntrinsicID(Main->Inst));
     for (auto i = II.begin(), e = II.end(); i != e; ++i) {
       auto ArgInfo = *i;
-      if (ArgInfo.isArgOrRet()) {
-        switch (ArgInfo.getRestriction()) {
-        case GenXIntrinsicInfo::FIXED4:
-          Fixed4 = &CI->getOperandUse(ArgInfo.getArgIdx());
-          break;
-        case GenXIntrinsicInfo::TWICEWIDTH:
-          TwiceWidth = &CI->getOperandUse(ArgInfo.getArgIdx());
-          break;
-        }
+      if (!ArgInfo.isArgOrRet())
+       continue;
+      switch (ArgInfo.getRestriction()) {
+      case GenXIntrinsicInfo::FIXED4:
+        Fixed4 = &Main->Inst->getOperandUse(ArgInfo.getArgIdx());
+        break;
+      case GenXIntrinsicInfo::TWICEWIDTH:
+        TwiceWidth = &Main->Inst->getOperandUse(ArgInfo.getArgIdx());
+        break;
       }
     }
-    return II.getExecSizeAllowedBits();
   }
-  return 0x3f;
+  return genx::getExecSizeAllowedBits(Main->Inst, ST);
 }
+
 /***********************************************************************
  * checkIfLongLongSupportNeeded: checks if an instruction requires
  *  target to support 64-bit integer operations
@@ -1382,9 +1328,7 @@ unsigned GenXLegalization::determineWidth(unsigned WholeWidth,
                                           unsigned StartIdx) {
   // Prepare to keep track of whether an instruction with a minimum width
   // (e.g. dp4) would be split too small, and whether we need to unbale.
-  unsigned ExecSizeAllowedBits = 0x3f;
-  if (auto Main = B.getMainInst())
-    ExecSizeAllowedBits = getExecSizeAllowedBits(Main->Inst);
+  unsigned ExecSizeAllowedBits = adjustTwiceWidthOrFixed4(B);
   unsigned MainInstMinWidth =
       1 << countTrailingZeros(ExecSizeAllowedBits, ZB_Undefined);
   // Determine the vector width that we need to split into.
@@ -2604,20 +2548,6 @@ static Value *createBitCastIfNeeded(Value *V, Type *NewTy,
   return Inst;
 }
 
-// Get type that represents OldTy as vector of NewScalarType.
-static Type *getNewVectorType(Type *OldTy, IntegerType *NewScalarType) {
-  IGC_ASSERT(OldTy->isIntOrIntVectorTy());
-  unsigned OldElemSize = OldTy->getScalarSizeInBits();
-  IGC_ASSERT(OldElemSize > 0);
-  auto *VTy = dyn_cast<IGCLLVM::FixedVectorType>(OldTy);
-  unsigned OldNumElems = VTy ? VTy->getNumElements() : 1;
-  unsigned NewElemSize = NewScalarType->getBitWidth();
-  if (OldElemSize * OldNumElems % NewElemSize)
-    return nullptr;
-  return IGCLLVM::FixedVectorType::get(NewScalarType,
-                         OldElemSize * OldNumElems / NewElemSize);
-}
-
 /***********************************************************************
  * transformMoveType : transform move bale to new integer type.
  *
@@ -2671,16 +2601,13 @@ Instruction *GenXLegalization::transformMoveType(Bale *B, IntegerType *FromTy,
     return nullptr;
   Region DstRgn = Wr ? makeRegionFromBaleInfo(Wr, BaleInfo()) : Region(Rd);
 
-  // Check that dst and src regions can be changed on new type.
-  if (SrcRgn.Indirect || DstRgn.Indirect || DstRgn.Mask)
-    return nullptr;
   // If destination region is not contiguous, changing element type can be achived
   // by conversion to 2D region, but we try to avoid it because such dst operands
   // are not supported in HW and require additional code for emulation.
   if (DstRgn.Stride != 1)
     return nullptr;
-  Type *NewSrcTy = getNewVectorType(Src->getType(), ToTy),
-       *NewDstTy = getNewVectorType(Dst->getType(), ToTy);
+  Type *NewSrcTy = genx::changeVectorType(Src->getType(), ToTy),
+       *NewDstTy = genx::changeVectorType(Dst->getType(), ToTy);
   if (!NewSrcTy || !NewDstTy)
     return nullptr;
 

IGC/VectorCompiler/lib/GenXCodeGen/GenXPostLegalization.cpp

@@ -138,7 +138,7 @@ bool GenXPostLegalization::runOnFunction(Function &F)
   // Run the vector decomposer for this function.
   Modified |= VD.run(DT);
   // Cleanup region reads and writes.
-  Modified |= simplifyRegionInsts(&F, DL);
+  Modified |= simplifyRegionInsts(&F, DL, ST);
   // Cleanup redundant global loads.
   Modified |= cleanupLoads(&F);
   // Cleanup constant loads.

IGC/VectorCompiler/lib/GenXCodeGen/GenXRegionUtils.cpp

@@ -24,10 +24,10 @@ SPDX-License-Identifier: MIT
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/Debug.h"
 #include <unordered_map>
-#include "Probe/Assertion.h"
 
 #include "llvmWrapper/IR/DerivedTypes.h"
 #include "llvmWrapper/Support/TypeSize.h"
@@ -922,14 +922,66 @@ static Instruction* simplifyConstIndirectRegion(Instruction* Inst) {
   return Inst;
 }
 
-static Value *simplifyRegionWrite(Instruction *Inst) {
-  IGC_ASSERT(GenXIntrinsic::isWrRegion(Inst));
-  Value *NewVal = Inst->getOperand(GenXIntrinsic::GenXRegion::NewValueOperandNum);
+// fold bitcast with wrregion:
+//                                  ==>  %oldval.cast = bitcast(%oldval)
+// %2 = bitcast(%1)                      %3 = wrregion(%oldval.cast, %1, ...)
+// %3 = wrregion(%oldval, %2, ...)       %2 = bitcast(%3)
+// so it can be baled later.
+static Value *simplifyBitCastWithRegionWrite(Instruction *WrR,
+                                             const DataLayout &DL,
+                                             const GenXSubtarget &ST) {
+  using namespace GenXIntrinsic::GenXRegion;
+  IGC_ASSERT(GenXIntrinsic::isWrRegion(WrR));
+  Value *NewVal = WrR->getOperand(NewValueOperandNum);
+  auto *BCI = dyn_cast<BitCastInst>(NewVal);
+  if (!BCI)
+    return nullptr;
+  if (WrR->hasOneUse() && GenXIntrinsic::isWritePredefReg(WrR->user_back()))
+    return nullptr;
+  auto *NewScalarTy = BCI->getSrcTy()->getScalarType();
+  // Do not change register category to predicate.
+  if (NewScalarTy->isIntegerTy(1))
+    return nullptr;
+  auto *OldVal = WrR->getOperand(OldValueOperandNum);
+  if (GenXIntrinsic::isReadWritePredefReg(OldVal))
+    return nullptr;
+  auto *NewVecTy = genx::changeVectorType(OldVal->getType(), NewScalarTy);
+  if (!NewVecTy)
+    return nullptr;
+  Region R = makeRegionFromBaleInfo(WrR, BaleInfo());
+  if (!R.changeElementType(NewScalarTy, &DL))
+    return nullptr;
+  // Transformation is not profitable for 2D regions or if it will require
+  // legalization.
+  if (R.is2D() || R.NumElements > llvm::PowerOf2Floor(
+                                      genx::getExecSizeAllowedBits(WrR, &ST)))
+    return nullptr;
+  IRBuilder<> IRB(WrR);
+  auto *OldValCast =
+      IRB.CreateBitCast(OldVal, NewVecTy, OldVal->getName() + ".cast");
+  auto *NewWrR = R.createWrRegion(OldValCast, BCI->getOperand(0),
+                                  WrR->getName(), WrR, WrR->getDebugLoc());
+  auto *NewBCI = IRB.CreateBitCast(NewWrR, WrR->getType(), BCI->getName());
+  return NewBCI;
+}
 
+static Value *simplifyRegionWrite(Instruction *WrR) {
+  using namespace GenXIntrinsic::GenXRegion;
+  IGC_ASSERT(GenXIntrinsic::isWrRegion(WrR));
+  Value *NewVal = WrR->getOperand(NewValueOperandNum);
+
+  // Replace C with B if R - whole region
+  // C = wrregion(A, B, R)
+  if (std::none_of(
+          WrR->user_begin(), WrR->user_end(),
+          [](auto *U) { return GenXIntrinsic::isWritePredefReg(U); }) &&
+      makeRegionFromBaleInfo(WrR, BaleInfo()).isWhole(WrR->getType()) &&
+      NewVal->getType() == WrR->getType())
+    return NewVal;
   // Replace C with A
   // C = wrregion(A, undef, R)
   if (isa<UndefValue>(NewVal))
-    return Inst->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
+    return WrR->getOperand(OldValueOperandNum);
 
   // When A and undef have the same type, replace C with A
   // B = rdregion(A, R)
@@ -941,29 +993,68 @@ static Value *simplifyRegionWrite(Instruction *Inst) {
   // C = wrregion(A, B, R)
   //
   if (GenXIntrinsic::isRdRegion(NewVal)) {
-    Instruction *B = cast<Instruction>(NewVal);
-    Region InnerR = makeRegionFromBaleInfo(B, BaleInfo());
-    Region OuterR = makeRegionFromBaleInfo(Inst, BaleInfo());
+    Instruction *RdR = cast<Instruction>(NewVal);
+    Region InnerR = makeRegionFromBaleInfo(RdR, BaleInfo());
+    Region OuterR = makeRegionFromBaleInfo(WrR, BaleInfo());
     if (OuterR != InnerR)
       return nullptr;
 
-    auto OldValB = B->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
-    if (GenXIntrinsic::isReadPredefReg(OldValB))
+    auto OldValRdR = RdR->getOperand(OldValueOperandNum);
+    if (GenXIntrinsic::isReadPredefReg(OldValRdR))
       return nullptr;
-    auto OldValC = Inst->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
-    if ((isa<UndefValue>(OldValC) &&
-         OldValB->getType() == OldValC->getType()) ||
-        OldValB == OldValC)
-      return OldValB;
+    auto OldValWrR = WrR->getOperand(OldValueOperandNum);
+    if ((isa<UndefValue>(OldValWrR) &&
+         OldValRdR->getType() == OldValWrR->getType()) ||
+        OldValRdR == OldValWrR)
+      return OldValRdR;
   }
-
   return nullptr;
 }
 
+// fold bitcast with rdregion:
+// %2 = rdregion(%1, ...)  ==>  %3 = bitcast(%1)
+// %3 = bitcast(%2)             %2 = rdregion(%3, ...)
+// so it can be baled later.
+static Value *simplifyBitCastFromRegionRead(BitCastInst *BCI,
+                                            const DataLayout &DL,
+                                            const GenXSubtarget &ST) {
+  using namespace GenXIntrinsic::GenXRegion;
+  Instruction *RdR = dyn_cast<Instruction>(BCI->getOperand(0));
+  if (!RdR || !GenXIntrinsic::isRdRegion(RdR) || !RdR->hasOneUse())
+    return nullptr;
+  auto *OldVal = RdR->getOperand(OldValueOperandNum);
+  if (GenXIntrinsic::isReadPredefReg(OldVal))
+    return nullptr;
+  auto *NewScalarTy = BCI->getDestTy()->getScalarType();
+  // Do not change register category to predicate.
+  if (NewScalarTy->isIntegerTy(1))
+    return nullptr;
+  auto *NewVecTy = genx::changeVectorType(OldVal->getType(), NewScalarTy);
+  if (!NewVecTy)
+    return nullptr;
+  Region R = makeRegionFromBaleInfo(RdR, BaleInfo());
+  if (!R.changeElementType(NewScalarTy, &DL))
+    return nullptr;
+  // Transformation is not profitable for 2D regions or if it will require
+  // legalization.
+  if (R.is2D() || R.NumElements > llvm::PowerOf2Floor(
+                                      genx::getExecSizeAllowedBits(RdR, &ST)))
+    return nullptr;
+  auto *NewBCI =
+      IRBuilder<>(BCI).CreateBitCast(OldVal, NewVecTy, BCI->getName());
+  auto *NewRdR =
+      R.createRdRegion(NewBCI, RdR->getName(), BCI, RdR->getDebugLoc());
+  return NewRdR;
+}
+
 static Value *simplifyRegionRead(Instruction *Inst) {
   IGC_ASSERT(GenXIntrinsic::isRdRegion(Inst));
   Value *Input = Inst->getOperand(GenXIntrinsic::GenXRegion::OldValueOperandNum);
-  if (isa<UndefValue>(Input))
+  if (!GenXIntrinsic::isReadPredefReg(Input) &&
+      makeRegionFromBaleInfo(Inst, BaleInfo()).isWhole(Input->getType()) &&
+      Input->getType() == Inst->getType())
+    return Input;
+  else if (isa<UndefValue>(Input))
     return UndefValue::get(Inst->getType());
   else if (auto C = dyn_cast<Constant>(Input)) {
     if (auto Splat = C->getSplatValue()) {
@@ -990,7 +1081,8 @@ static Value *simplifyRegionRead(Instruction *Inst) {
 }
 
 // Simplify a region read or write.
-Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL) {
+Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL,
+                                      const GenXSubtarget *ST) {
   if (Inst->use_empty())
     return nullptr;
 
@@ -1013,11 +1105,17 @@ Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL) {
   if (Constant *C = ConstantFoldGenX(Inst, *DL))
     return C;
 
+  if (auto *BCI = dyn_cast<BitCastInst>(Inst); BCI && DL && ST)
+    return simplifyBitCastFromRegionRead(BCI, *DL, *ST);
   ID = GenXIntrinsic::getGenXIntrinsicID(Inst);
   switch (ID) {
   case GenXIntrinsic::genx_wrregionf:
   case GenXIntrinsic::genx_wrregioni:
-    return simplifyRegionWrite(Inst);
+    if (auto *Res = simplifyRegionWrite(Inst))
+      return Res;
+    if (DL && ST)
+      return simplifyBitCastWithRegionWrite(Inst, *DL, *ST);
+    break;
   case GenXIntrinsic::genx_rdregionf:
   case GenXIntrinsic::genx_rdregioni:
     return simplifyRegionRead(Inst);
@@ -1027,12 +1125,13 @@ Value *llvm::genx::simplifyRegionInst(Instruction *Inst, const DataLayout *DL) {
   return nullptr;
 }
 
-bool llvm::genx::simplifyRegionInsts(Function *F, const DataLayout *DL) {
+bool llvm::genx::simplifyRegionInsts(Function *F, const DataLayout *DL,
+                                     const GenXSubtarget *ST) {
   bool Changed = false;
   for (auto &BB : F->getBasicBlockList()) {
     for (auto I = BB.begin(); I != BB.end();) {
       Instruction *Inst = &*I++;
-      if (auto V = simplifyRegionInst(Inst, DL)) {
+      if (auto V = simplifyRegionInst(Inst, DL, ST)) {
         Inst->replaceAllUsesWith(V);
         Inst->eraseFromParent();
         Changed = true;

IGC/VectorCompiler/lib/GenXCodeGen/GenXRegionUtils.h

@@ -162,8 +162,10 @@ inline raw_ostream &operator<<(raw_ostream &OS, const RdWrRegionSequence &RWS) {
   return OS;
 }
 
-Value *simplifyRegionInst(Instruction *Inst, const DataLayout *DL);
-bool simplifyRegionInsts(Function *F, const DataLayout *DL);
+Value *simplifyRegionInst(Instruction *Inst, const DataLayout *DL = nullptr,
+                          const GenXSubtarget *ST = nullptr);
+bool simplifyRegionInsts(Function *F, const DataLayout *DL = nullptr,
+                         const GenXSubtarget *ST = nullptr);
 
 bool cleanupLoads(Function *F);