diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 5a4986a729a350..34855bf766fec5 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -7756,91 +7756,90 @@ static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl &Inputs, /// Returns the scalar element that will make up the i'th /// element of the result of the vector shuffle. -static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG, - unsigned Depth) { +static SDValue getShuffleScalarElt(SDValue Op, unsigned Index, + SelectionDAG &DAG, unsigned Depth) { if (Depth >= SelectionDAG::MaxRecursionDepth) return SDValue(); // Limit search depth. - SDValue V = SDValue(N, 0); - EVT VT = V.getValueType(); - unsigned Opcode = V.getOpcode(); + EVT VT = Op.getValueType(); + unsigned Opcode = Op.getOpcode(); + unsigned NumElems = VT.getVectorNumElements(); // Recurse into ISD::VECTOR_SHUFFLE node to find scalars. - if (auto *SV = dyn_cast(N)) { + if (auto *SV = dyn_cast(Op)) { int Elt = SV->getMaskElt(Index); if (Elt < 0) return DAG.getUNDEF(VT.getVectorElementType()); - unsigned NumElems = VT.getVectorNumElements(); - SDValue NewV = - (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1); - return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth + 1); + SDValue Src = (Elt < (int)NumElems) ? SV->getOperand(0) : SV->getOperand(1); + return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1); } // Recurse into target specific vector shuffles to find scalars. if (isTargetShuffle(Opcode)) { - MVT ShufVT = V.getSimpleValueType(); + MVT ShufVT = VT.getSimpleVT(); MVT ShufSVT = ShufVT.getVectorElementType(); int NumElems = (int)ShufVT.getVectorNumElements(); SmallVector ShuffleMask; SmallVector ShuffleOps; bool IsUnary; - if (!getTargetShuffleMask(N, ShufVT, true, ShuffleOps, ShuffleMask, - IsUnary)) + if (!getTargetShuffleMask(Op.getNode(), ShufVT, true, ShuffleOps, + ShuffleMask, IsUnary)) return SDValue(); int Elt = ShuffleMask[Index]; if (Elt == SM_SentinelZero) - return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(N), ShufSVT) - : DAG.getConstantFP(+0.0, SDLoc(N), ShufSVT); + return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(Op), ShufSVT) + : DAG.getConstantFP(+0.0, SDLoc(Op), ShufSVT); if (Elt == SM_SentinelUndef) return DAG.getUNDEF(ShufSVT); assert(0 <= Elt && Elt < (2 * NumElems) && "Shuffle index out of range"); - SDValue NewV = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1]; - return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth + 1); + SDValue Src = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1]; + return getShuffleScalarElt(Src, Elt % NumElems, DAG, Depth + 1); } // Recurse into insert_subvector base/sub vector to find scalars. if (Opcode == ISD::INSERT_SUBVECTOR && - isa(N->getOperand(2))) { - SDValue Vec = N->getOperand(0); - SDValue Sub = N->getOperand(1); + isa(Op.getOperand(2))) { + SDValue Vec = Op.getOperand(0); + SDValue Sub = Op.getOperand(1); EVT SubVT = Sub.getValueType(); unsigned NumSubElts = SubVT.getVectorNumElements(); - uint64_t SubIdx = N->getConstantOperandVal(2); + uint64_t SubIdx = Op.getConstantOperandVal(2); if (SubIdx <= Index && Index < (SubIdx + NumSubElts)) - return getShuffleScalarElt(Sub.getNode(), Index - SubIdx, DAG, Depth + 1); - return getShuffleScalarElt(Vec.getNode(), Index, DAG, Depth + 1); + return getShuffleScalarElt(Sub, Index - SubIdx, DAG, Depth + 1); + return getShuffleScalarElt(Vec, Index, DAG, Depth + 1); } // Recurse into extract_subvector src vector to find scalars. if (Opcode == ISD::EXTRACT_SUBVECTOR && - isa(N->getOperand(1))) { - SDValue Src = N->getOperand(0); - uint64_t SrcIdx = N->getConstantOperandVal(1); - return getShuffleScalarElt(Src.getNode(), Index + SrcIdx, DAG, Depth + 1); + isa(Op.getOperand(1))) { + SDValue Src = Op.getOperand(0); + uint64_t SrcIdx = Op.getConstantOperandVal(1); + return getShuffleScalarElt(Src, Index + SrcIdx, DAG, Depth + 1); } - // Actual nodes that may contain scalar elements + // We only peek through bitcasts of the same vector width. if (Opcode == ISD::BITCAST) { - V = V.getOperand(0); - EVT SrcVT = V.getValueType(); - unsigned NumElems = VT.getVectorNumElements(); - - if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems) - return SDValue(); + SDValue Src = Op.getOperand(0); + EVT SrcVT = Src.getValueType(); + if (SrcVT.isVector() && SrcVT.getVectorNumElements() == NumElems) + return getShuffleScalarElt(Src, Index, DAG, Depth + 1); + return SDValue(); } - if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) - return (Index == 0) ? V.getOperand(0) + // Actual nodes that may contain scalar elements + + if (Opcode == ISD::SCALAR_TO_VECTOR) + return (Index == 0) ? Op.getOperand(0) : DAG.getUNDEF(VT.getVectorElementType()); - if (V.getOpcode() == ISD::BUILD_VECTOR) - return V.getOperand(Index); + if (Opcode == ISD::BUILD_VECTOR) + return Op.getOperand(Index); return SDValue(); } @@ -8492,13 +8491,13 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef Elts, // Combine a vector ops (shuffles etc.) that is equal to build_vector load1, // load2, load3, load4, <0, 1, 2, 3> into a vector load if the load addresses // are consecutive, non-overlapping, and in the right order. -static SDValue combineToConsecutiveLoads(EVT VT, SDNode *N, const SDLoc &DL, +static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool isAfterLegalize) { SmallVector Elts; for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { - if (SDValue Elt = getShuffleScalarElt(N, i, DAG, 0)) { + if (SDValue Elt = getShuffleScalarElt(Op, i, DAG, 0)) { Elts.push_back(Elt); continue; } @@ -36396,7 +36395,8 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG, } // Attempt to combine into a vector load/broadcast. - if (SDValue LD = combineToConsecutiveLoads(VT, N, dl, DAG, Subtarget, true)) + if (SDValue LD = combineToConsecutiveLoads(VT, SDValue(N, 0), dl, DAG, + Subtarget, true)) return LD; // For AVX2, we sometimes want to combine