diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 941de90ff4b76..2765887fa1809 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1284,10 +1284,12 @@ class SelectionDAG { /// vector op and fill the end of the resulting vector with UNDEFS. SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); - /// Return true if LD is loading 'Bytes' bytes from a location that is 'Dist' - /// units away from the location that the 'Base' load is loading from. - bool isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, - unsigned Bytes, int Dist) const; + /// Return true if loads are next to each other and can be + /// merged. Check that both are nonvolatile and if LD is loading + /// 'Bytes' bytes from a location that is 'Dist' units away from the + /// location that the 'Base' load is loading from. + bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, + unsigned Bytes, int Dist) const; /// Infer alignment of a load / store address. Return 0 if /// it cannot be inferred. diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 09c7971c97db8..8544c27263b2c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -7251,14 +7251,9 @@ SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) { LD1->getAddressSpace() != LD2->getAddressSpace()) return SDValue(); EVT LD1VT = LD1->getValueType(0); - - if (ISD::isNON_EXTLoad(LD2) && - LD2->hasOneUse() && - // If both are volatile this would reduce the number of volatile loads. - // If one is volatile it might be ok, but play conservative and bail out. - !LD1->isVolatile() && - !LD2->isVolatile() && - DAG.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1)) { + unsigned LD1Bytes = LD1VT.getSizeInBits() / 8; + if (ISD::isNON_EXTLoad(LD2) && LD2->hasOneUse() && + DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1)) { unsigned Align = LD1->getAlignment(); unsigned NewAlign = DAG.getDataLayout().getABITypeAlignment( VT.getTypeForEVT(*DAG.getContext())); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index ebf49b985a3ce..46986d7e0adb7 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -6959,12 +6959,12 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { EVT::getVectorVT(*getContext(), EltVT, ResNE), Scalars); } - -/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a -/// location that is 'Dist' units away from the location that the 'Base' load -/// is loading from. -bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, - unsigned Bytes, int Dist) const { +bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, + LoadSDNode *Base, + unsigned Bytes, + int Dist) const { + if (LD->isVolatile() || Base->isVolatile()) + return false; if (LD->getChain() != Base->getChain()) return false; EVT VT = LD->getValueType(0); diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 8b0ad77702139..1032137e8f6b5 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -5736,9 +5736,9 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef Elts, if (LoadMask[i]) { SDValue Elt = peekThroughBitcasts(Elts[i]); LoadSDNode *LD = cast(Elt); - if (!DAG.isConsecutiveLoad(LD, LDBase, - Elt.getValueType().getStoreSizeInBits() / 8, - i - FirstLoadedElt)) { + if (!DAG.areNonVolatileConsecutiveLoads( + LD, LDBase, Elt.getValueType().getStoreSizeInBits() / 8, + i - FirstLoadedElt)) { IsConsecutiveLoad = false; IsConsecutiveLoadWithZeros = false; break; @@ -5749,10 +5749,10 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef Elts, } auto CreateLoad = [&DAG, &DL](EVT VT, LoadSDNode *LDBase) { - SDValue NewLd = DAG.getLoad(VT, DL, LDBase->getChain(), - LDBase->getBasePtr(), LDBase->getPointerInfo(), - LDBase->isVolatile(), LDBase->isNonTemporal(), - LDBase->isInvariant(), LDBase->getAlignment()); + SDValue NewLd = DAG.getLoad( + VT, DL, LDBase->getChain(), LDBase->getBasePtr(), + LDBase->getPointerInfo(), false /*LDBase->isVolatile()*/, + LDBase->isNonTemporal(), LDBase->isInvariant(), LDBase->getAlignment()); if (LDBase->hasAnyUseOfValue(1)) { SDValue NewChain =