From 05e7d8d6ce0e4f87aa1c4490d9f843360a6b2ca5 Mon Sep 17 00:00:00 2001 From: Matt Arsenault Date: Fri, 14 Feb 2020 13:22:53 -0500 Subject: [PATCH] TTI: Add addrspace parameters to memcpy lowering functions --- .../include/llvm/Analysis/TargetTransformInfo.h | 17 +++++++++++++++-- .../llvm/Analysis/TargetTransformInfoImpl.h | 3 +++ llvm/lib/Analysis/TargetTransformInfo.cpp | 11 +++++++++-- .../lib/Transforms/Utils/LowerMemIntrinsics.cpp | 15 ++++++++------- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 1387b90307f7f..ca6892b14ef3b 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1069,6 +1069,7 @@ class TargetTransformInfo { /// \returns The type to use in a loop expansion of a memcpy call. Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAddrSpace, unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const; /// \param[out] OpsOut The operand types to copy RemainingBytes of memory. @@ -1080,6 +1081,8 @@ class TargetTransformInfo { void getMemcpyLoopResidualLoweringType(SmallVectorImpl &OpsOut, LLVMContext &Context, unsigned RemainingBytes, + unsigned SrcAddrSpace, + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const; @@ -1382,11 +1385,15 @@ class TargetTransformInfo::Concept { virtual Value *getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, Type *ExpectedType) = 0; virtual Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAddrSpace, + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const = 0; virtual void getMemcpyLoopResidualLoweringType( SmallVectorImpl &OpsOut, LLVMContext &Context, - unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const = 0; + unsigned RemainingBytes, + unsigned SrcAddrSpace, unsigned DestAddrSpace, + unsigned SrcAlign, unsigned DestAlign) const = 0; virtual bool areInlineCompatible(const Function *Caller, const Function *Callee) const = 0; virtual bool @@ -1830,16 +1837,22 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept { return Impl.getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); } Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAddrSpace, unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const override { - return Impl.getMemcpyLoopLoweringType(Context, Length, SrcAlign, DestAlign); + return Impl.getMemcpyLoopLoweringType(Context, Length, + SrcAddrSpace, DestAddrSpace, + SrcAlign, DestAlign); } void getMemcpyLoopResidualLoweringType(SmallVectorImpl &OpsOut, LLVMContext &Context, unsigned RemainingBytes, + unsigned SrcAddrSpace, + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const override { Impl.getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, + SrcAddrSpace, DestAddrSpace, SrcAlign, DestAlign); } bool areInlineCompatible(const Function *Caller, diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index d825a0796c5d3..2acb88a6a83d1 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -543,6 +543,7 @@ class TargetTransformInfoImplBase { } Type *getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAddrSpace, unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const { return Type::getInt8Ty(Context); } @@ -550,6 +551,8 @@ class TargetTransformInfoImplBase { void getMemcpyLoopResidualLoweringType(SmallVectorImpl &OpsOut, LLVMContext &Context, unsigned RemainingBytes, + unsigned SrcAddrSpace, + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const { for (unsigned i = 0; i != RemainingBytes; ++i) diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index a1b30fc8f12b1..89b1783a9a007 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -776,16 +776,23 @@ Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context, Value *Length, + unsigned SrcAddrSpace, + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const { - return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign, + return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace, + DestAddrSpace, SrcAlign, DestAlign); } void TargetTransformInfo::getMemcpyLoopResidualLoweringType( SmallVectorImpl &OpsOut, LLVMContext &Context, - unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const { + unsigned RemainingBytes, + unsigned SrcAddrSpace, + unsigned DestAddrSpace, + unsigned SrcAlign, unsigned DestAlign) const { TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, + SrcAddrSpace, DestAddrSpace, SrcAlign, DestAlign); } diff --git a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp index 0cc085dc366c6..13c3b23a1c0ff 100644 --- a/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp +++ b/llvm/lib/Transforms/Utils/LowerMemIntrinsics.cpp @@ -36,16 +36,16 @@ void llvm::createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr, Function *ParentFunc = PreLoopBB->getParent(); LLVMContext &Ctx = PreLoopBB->getContext(); + unsigned SrcAS = cast(SrcAddr->getType())->getAddressSpace(); + unsigned DstAS = cast(DstAddr->getType())->getAddressSpace(); + Type *TypeOfCopyLen = CopyLen->getType(); Type *LoopOpType = - TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAlign, DestAlign); + TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAS, DstAS, SrcAlign, DestAlign); unsigned LoopOpSize = getLoopOperandSizeInBytes(LoopOpType); uint64_t LoopEndCount = CopyLen->getZExtValue() / LoopOpSize; - unsigned SrcAS = cast(SrcAddr->getType())->getAddressSpace(); - unsigned DstAS = cast(DstAddr->getType())->getAddressSpace(); - if (LoopEndCount != 0) { // Split PostLoopBB = PreLoopBB->splitBasicBlock(InsertBefore, "memcpy-split"); @@ -99,6 +99,7 @@ void llvm::createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr, SmallVector RemainingOps; TTI.getMemcpyLoopResidualLoweringType(RemainingOps, Ctx, RemainingBytes, + SrcAS, DstAS, SrcAlign, DestAlign); for (auto OpTy : RemainingOps) { @@ -144,15 +145,15 @@ void llvm::createMemCpyLoopUnknownSize(Instruction *InsertBefore, Function *ParentFunc = PreLoopBB->getParent(); LLVMContext &Ctx = PreLoopBB->getContext(); + unsigned SrcAS = cast(SrcAddr->getType())->getAddressSpace(); + unsigned DstAS = cast(DstAddr->getType())->getAddressSpace(); Type *LoopOpType = - TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAlign, DestAlign); + TTI.getMemcpyLoopLoweringType(Ctx, CopyLen, SrcAS, DstAS, SrcAlign, DestAlign); unsigned LoopOpSize = getLoopOperandSizeInBytes(LoopOpType); IRBuilder<> PLBuilder(PreLoopBB->getTerminator()); - unsigned SrcAS = cast(SrcAddr->getType())->getAddressSpace(); - unsigned DstAS = cast(DstAddr->getType())->getAddressSpace(); PointerType *SrcOpType = PointerType::get(LoopOpType, SrcAS); PointerType *DstOpType = PointerType::get(LoopOpType, DstAS); if (SrcAddr->getType() != SrcOpType) {