diff --git a/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h new file mode 100644 index 0000000000000..06a44f1885656 --- /dev/null +++ b/flang/include/flang/Optimizer/CodeGen/FIROpPatterns.h @@ -0,0 +1,248 @@ +//===-- FIROpPatterns.h -- FIR operation conversion patterns ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H +#define FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H + +#include "flang/Optimizer/CodeGen/TypeConverter.h" +#include "mlir/Conversion/LLVMCommon/Pattern.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" + +namespace fir { + +struct FIRToLLVMPassOptions; + +static constexpr unsigned defaultAddressSpace = 0u; + +class ConvertFIRToLLVMPattern : public mlir::ConvertToLLVMPattern { +public: + ConvertFIRToLLVMPattern(llvm::StringRef rootOpName, + mlir::MLIRContext *context, + const fir::LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, + mlir::PatternBenefit benefit = 1); + +protected: + mlir::Type convertType(mlir::Type ty) const { + return lowerTy().convertType(ty); + } + + // Convert FIR type to LLVM without turning fir.box into memory + // reference. + mlir::Type convertObjectType(mlir::Type firType) const; + + mlir::LLVM::ConstantOp + genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int value) const; + + mlir::LLVM::ConstantOp + genConstantOffset(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + int offset) const; + + /// Perform an extension or truncation as needed on an integer value. Lowering + /// to the specific target may involve some sign-extending or truncation of + /// values, particularly to fit them from abstract box types to the + /// appropriate reified structures. + mlir::Value integerCast(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + mlir::Type ty, mlir::Value val) const; + struct TypePair { + mlir::Type fir; + mlir::Type llvm; + }; + + TypePair getBoxTypePair(mlir::Type firBoxTy) const; + + /// Construct code sequence to extract the specific value from a `fir.box`. + mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy, + mlir::Value box, mlir::Type resultTy, + mlir::ConversionPatternRewriter &rewriter, + int boxValue) const; + + /// Method to construct code sequence to get the triple for dimension `dim` + /// from a box. + llvm::SmallVector + getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, + TypePair boxTy, mlir::Value box, mlir::Value dim, + mlir::ConversionPatternRewriter &rewriter) const; + + llvm::SmallVector + getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, + TypePair boxTy, mlir::Value box, int dim, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::Value dim, int off, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + int dim, int off, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value getStrideFromBox(mlir::Location loc, TypePair boxTy, + mlir::Value box, unsigned dim, + mlir::ConversionPatternRewriter &rewriter) const; + + /// Read base address from a fir.box. Returned address has type ty. + mlir::Value + getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + mlir::Value + getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy, + mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + // Get the element type given an LLVM type that is of the form + // (array|struct|vector)+ and the provided indexes. + mlir::Type getBoxEleTy(mlir::Type type, + llvm::ArrayRef indexes) const; + + // Return LLVM type of the object described by a fir.box of \p boxType. + mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const; + + /// Read the address of the type descriptor from a box. + mlir::Value + loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const; + + // Load the attribute from the \p box and perform a check against \p maskValue + // The final comparison is implemented as `(attribute & maskValue) != 0`. + mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy, + mlir::Value box, + mlir::ConversionPatternRewriter &rewriter, + unsigned maskValue) const; + + template + mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty, + mlir::ConversionPatternRewriter &rewriter, + mlir::Value base, ARGS... args) const { + llvm::SmallVector cv = {args...}; + auto llvmPtrTy = + mlir::LLVM::LLVMPointerType::get(ty.getContext(), /*addressSpace=*/0); + return rewriter.create(loc, llvmPtrTy, ty, base, cv); + } + + // Find the Block in which the alloca should be inserted. + // The order to recursively find the proper block: + // 1. An OpenMP Op that will be outlined. + // 2. A LLVMFuncOp + // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp + mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) const; + + // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the + // allocation address space provided for the architecture in the DataLayout + // specification. If the address space is different from the devices + // program address space we perform a cast. In the case of most architectures + // the program and allocation address space will be the default of 0 and no + // cast will be emitted. + mlir::Value + genAllocaAndAddrCastWithType(mlir::Location loc, mlir::Type llvmObjectTy, + unsigned alignment, + mlir::ConversionPatternRewriter &rewriter) const; + + const fir::LLVMTypeConverter &lowerTy() const { + return *static_cast( + this->getTypeConverter()); + } + + void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op, + mlir::Type baseFIRType, mlir::Type accessFIRType, + mlir::LLVM::GEPOp gep) const { + lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep); + } + + unsigned + getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) const; + + unsigned + getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) const; + + const fir::FIRToLLVMPassOptions &options; + + using ConvertToLLVMPattern::match; + using ConvertToLLVMPattern::matchAndRewrite; +}; + +template +class FIROpConversion : public ConvertFIRToLLVMPattern { +public: + using OpAdaptor = typename SourceOp::Adaptor; + + explicit FIROpConversion(const LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, + mlir::PatternBenefit benefit = 1) + : ConvertFIRToLLVMPattern(SourceOp::getOperationName(), + &typeConverter.getContext(), typeConverter, + options, benefit) {} + + /// Wrappers around the RewritePattern methods that pass the derived op type. + void rewrite(mlir::Operation *op, mlir::ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const final { + rewrite(mlir::cast(op), + OpAdaptor(operands, mlir::cast(op)), rewriter); + } + mlir::LogicalResult match(mlir::Operation *op) const final { + return match(mlir::cast(op)); + } + mlir::LogicalResult + matchAndRewrite(mlir::Operation *op, mlir::ArrayRef operands, + mlir::ConversionPatternRewriter &rewriter) const final { + return matchAndRewrite(mlir::cast(op), + OpAdaptor(operands, mlir::cast(op)), + rewriter); + } + + /// Rewrite and Match methods that operate on the SourceOp type. These must be + /// overridden by the derived pattern class. + virtual mlir::LogicalResult match(SourceOp op) const { + llvm_unreachable("must override match or matchAndRewrite"); + } + virtual void rewrite(SourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + llvm_unreachable("must override rewrite or matchAndRewrite"); + } + virtual mlir::LogicalResult + matchAndRewrite(SourceOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const { + if (mlir::failed(match(op))) + return mlir::failure(); + rewrite(op, adaptor, rewriter); + return mlir::success(); + } + +private: + using ConvertFIRToLLVMPattern::matchAndRewrite; + using ConvertToLLVMPattern::match; +}; + +/// FIR conversion pattern template +template +class FIROpAndTypeConversion : public FIROpConversion { +public: + using FIROpConversion::FIROpConversion; + using OpAdaptor = typename FromOp::Adaptor; + + mlir::LogicalResult + matchAndRewrite(FromOp op, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const final { + mlir::Type ty = this->convertType(op.getType()); + return doRewrite(op, ty, adaptor, rewriter); + } + + virtual mlir::LogicalResult + doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor, + mlir::ConversionPatternRewriter &rewriter) const = 0; +}; + +} // namespace fir + +#endif // FORTRAN_OPTIMIZER_CODEGEN_FIROPPATTERNS_H diff --git a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h index 396c136392555..79b3bfe4e80e0 100644 --- a/flang/include/flang/Optimizer/CodeGen/TypeConverter.h +++ b/flang/include/flang/Optimizer/CodeGen/TypeConverter.h @@ -94,8 +94,8 @@ class LLVMTypeConverter : public mlir::LLVMTypeConverter { // to LLVM IR dialect here. // // fir.complex | std.complex --> llvm<"{t,t}"> - template mlir::Type convertComplexType(C cmplx) const { - LLVM_DEBUG(llvm::dbgs() << "type convert: " << cmplx << '\n'); + template + mlir::Type convertComplexType(C cmplx) const { auto eleTy = cmplx.getElementType(); return convertType(specifics->complexMemoryType(eleTy)); } diff --git a/flang/lib/Optimizer/CodeGen/CMakeLists.txt b/flang/lib/Optimizer/CodeGen/CMakeLists.txt index 175ab9fefda2a..879bc28d017a3 100644 --- a/flang/lib/Optimizer/CodeGen/CMakeLists.txt +++ b/flang/lib/Optimizer/CodeGen/CMakeLists.txt @@ -3,6 +3,7 @@ add_flang_library(FIRCodeGen CGOps.cpp CodeGen.cpp CodeGenOpenMP.cpp + FIROpPatterns.cpp PreCGRewrite.cpp TBAABuilder.cpp Target.cpp diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp index faf90ef6b50a9..06ce84f1543a3 100644 --- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp +++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp @@ -14,6 +14,8 @@ #include "CGOps.h" #include "flang/Optimizer/CodeGen/CodeGenOpenMP.h" +#include "flang/Optimizer/CodeGen/FIROpPatterns.h" +#include "flang/Optimizer/CodeGen/TypeConverter.h" #include "flang/Optimizer/Dialect/FIRAttr.h" #include "flang/Optimizer/Dialect/FIROps.h" #include "flang/Optimizer/Dialect/FIRType.h" @@ -58,42 +60,14 @@ namespace fir { #define DEBUG_TYPE "flang-codegen" -// fir::LLVMTypeConverter for converting to LLVM IR dialect types. -#include "flang/Optimizer/CodeGen/TypeConverter.h" - // TODO: This should really be recovered from the specified target. static constexpr unsigned defaultAlign = 8; -static constexpr unsigned defaultAddressSpace = 0u; /// `fir.box` attribute values as defined for CFI_attribute_t in /// flang/ISO_Fortran_binding.h. static constexpr unsigned kAttrPointer = CFI_attribute_pointer; static constexpr unsigned kAttrAllocatable = CFI_attribute_allocatable; -static inline unsigned -getAllocaAddressSpace(mlir::ConversionPatternRewriter &rewriter) { - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - assert(parentOp != nullptr && - "expected insertion block to have parent operation"); - if (auto module = parentOp->getParentOfType()) - if (mlir::Attribute addrSpace = - mlir::DataLayout(module).getAllocaMemorySpace()) - return llvm::cast(addrSpace).getUInt(); - return defaultAddressSpace; -} - -static inline unsigned -getProgramAddressSpace(mlir::ConversionPatternRewriter &rewriter) { - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - assert(parentOp != nullptr && - "expected insertion block to have parent operation"); - if (auto module = parentOp->getParentOfType()) - if (mlir::Attribute addrSpace = - mlir::DataLayout(module).getProgramMemorySpace()) - return llvm::cast(addrSpace).getUInt(); - return defaultAddressSpace; -} - static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context, unsigned addressSpace = 0) { return mlir::LLVM::LLVMPointerType::get(context, addressSpace); @@ -151,333 +125,9 @@ static unsigned getLenParamFieldId(mlir::Type ty) { return getTypeDescFieldId(ty) + 1; } -namespace { -/// FIR conversion pattern template -template -class FIROpConversion : public mlir::ConvertOpToLLVMPattern { -public: - explicit FIROpConversion(const fir::LLVMTypeConverter &lowering, - const fir::FIRToLLVMPassOptions &options) - : mlir::ConvertOpToLLVMPattern(lowering), options(options) {} - -protected: - mlir::Type convertType(mlir::Type ty) const { - return lowerTy().convertType(ty); - } - - // Convert FIR type to LLVM without turning fir.box into memory - // reference. - mlir::Type convertObjectType(mlir::Type firType) const { - if (auto boxTy = firType.dyn_cast()) - return lowerTy().convertBoxTypeAsStruct(boxTy); - return lowerTy().convertType(firType); - } - - mlir::LLVM::ConstantOp - genI32Constant(mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, - int value) const { - mlir::Type i32Ty = rewriter.getI32Type(); - mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value); - return rewriter.create(loc, i32Ty, attr); - } - - mlir::LLVM::ConstantOp - genConstantOffset(mlir::Location loc, - mlir::ConversionPatternRewriter &rewriter, - int offset) const { - mlir::Type ity = lowerTy().offsetType(); - mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset); - return rewriter.create(loc, ity, cattr); - } - - /// Perform an extension or truncation as needed on an integer value. Lowering - /// to the specific target may involve some sign-extending or truncation of - /// values, particularly to fit them from abstract box types to the - /// appropriate reified structures. - mlir::Value integerCast(mlir::Location loc, - mlir::ConversionPatternRewriter &rewriter, - mlir::Type ty, mlir::Value val) const { - auto valTy = val.getType(); - // If the value was not yet lowered, lower its type so that it can - // be used in getPrimitiveTypeSizeInBits. - if (!valTy.isa()) - valTy = convertType(valTy); - auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty); - auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy); - if (toSize < fromSize) - return rewriter.create(loc, ty, val); - if (toSize > fromSize) - return rewriter.create(loc, ty, val); - return val; - } - - struct TypePair { - mlir::Type fir; - mlir::Type llvm; - }; - - TypePair getBoxTypePair(mlir::Type firBoxTy) const { - mlir::Type llvmBoxTy = lowerTy().convertBoxTypeAsStruct( - mlir::cast(firBoxTy)); - return TypePair{firBoxTy, llvmBoxTy}; - } - - /// Construct code sequence to extract the specific value from a `fir.box`. - mlir::Value getValueFromBox(mlir::Location loc, TypePair boxTy, - mlir::Value box, mlir::Type resultTy, - mlir::ConversionPatternRewriter &rewriter, - int boxValue) const { - if (box.getType().isa()) { - auto pty = ::getLlvmPtrType(resultTy.getContext()); - auto p = rewriter.create( - loc, pty, boxTy.llvm, box, - llvm::ArrayRef{0, boxValue}); - auto loadOp = rewriter.create(loc, resultTy, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - return rewriter.create(loc, box, boxValue); - } - - /// Method to construct code sequence to get the triple for dimension `dim` - /// from a box. - llvm::SmallVector - getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, - TypePair boxTy, mlir::Value box, mlir::Value dim, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Value l0 = - loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); - mlir::Value l1 = - loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); - mlir::Value l2 = - loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); - return {l0, l1, l2}; - } - - llvm::SmallVector - getDimsFromBox(mlir::Location loc, llvm::ArrayRef retTys, - TypePair boxTy, mlir::Value box, int dim, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Value l0 = - getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); - mlir::Value l1 = - getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); - mlir::Value l2 = - getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); - return {l0, l1, l2}; - } - - mlir::Value - loadDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::Value dim, int off, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter) const { - assert(box.getType().isa() && - "descriptor inquiry with runtime dim can only be done on descriptor " - "in memory"); - mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, - static_cast(kDimsPosInBox), dim, off); - auto loadOp = rewriter.create(loc, ty, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - - mlir::Value - getDimFieldFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - int dim, int off, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter) const { - if (box.getType().isa()) { - mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, - static_cast(kDimsPosInBox), dim, off); - auto loadOp = rewriter.create(loc, ty, p); - attachTBAATag(loadOp, boxTy.fir, nullptr, p); - return loadOp; - } - return rewriter.create( - loc, box, llvm::ArrayRef{kDimsPosInBox, dim, off}); - } - - mlir::Value - getStrideFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - unsigned dim, - mlir::ConversionPatternRewriter &rewriter) const { - auto idxTy = lowerTy().indexType(); - return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy, - rewriter); - } - - /// Read base address from a fir.box. Returned address has type ty. - mlir::Value - getBaseAddrFromBox(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext()); - return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox); - } - - mlir::Value - getElementSizeFromBox(mlir::Location loc, mlir::Type resultTy, TypePair boxTy, - mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - return getValueFromBox(loc, boxTy, box, resultTy, rewriter, - kElemLenPosInBox); - } - - // Get the element type given an LLVM type that is of the form - // (array|struct|vector)+ and the provided indexes. - static mlir::Type getBoxEleTy(mlir::Type type, - llvm::ArrayRef indexes) { - for (unsigned i : indexes) { - if (auto t = type.dyn_cast()) { - assert(!t.isOpaque() && i < t.getBody().size()); - type = t.getBody()[i]; - } else if (auto t = type.dyn_cast()) { - type = t.getElementType(); - } else if (auto t = type.dyn_cast()) { - type = t.getElementType(); - } else { - fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()), - "request for invalid box element type"); - } - } - return type; - } - - // Return LLVM type of the object described by a fir.box of \p boxType. - mlir::Type getLlvmObjectTypeFromBoxType(mlir::Type boxType) const { - mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType); - assert(objectType && "boxType must be a box type"); - return this->convertType(objectType); - } - - /// Read the address of the type descriptor from a box. - mlir::Value - loadTypeDescAddress(mlir::Location loc, TypePair boxTy, mlir::Value box, - mlir::ConversionPatternRewriter &rewriter) const { - unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir); - mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext()); - return getValueFromBox(loc, boxTy, box, tdescType, rewriter, - typeDescFieldId); - } - - // Load the attribute from the \p box and perform a check against \p maskValue - // The final comparison is implemented as `(attribute & maskValue) != 0`. - mlir::Value genBoxAttributeCheck(mlir::Location loc, TypePair boxTy, - mlir::Value box, - mlir::ConversionPatternRewriter &rewriter, - unsigned maskValue) const { - mlir::Type attrTy = rewriter.getI32Type(); - mlir::Value attribute = - getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox); - mlir::LLVM::ConstantOp attrMask = - genConstantOffset(loc, rewriter, maskValue); - auto maskRes = - rewriter.create(loc, attrTy, attribute, attrMask); - mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0); - return rewriter.create( - loc, mlir::LLVM::ICmpPredicate::ne, maskRes, c0); - } - - template - mlir::LLVM::GEPOp genGEP(mlir::Location loc, mlir::Type ty, - mlir::ConversionPatternRewriter &rewriter, - mlir::Value base, ARGS... args) const { - llvm::SmallVector cv = {args...}; - auto llvmPtrTy = ::getLlvmPtrType(ty.getContext()); - return rewriter.create(loc, llvmPtrTy, ty, base, cv); - } - - // Find the Block in which the alloca should be inserted. - // The order to recursively find the proper block: - // 1. An OpenMP Op that will be outlined. - // 2. A LLVMFuncOp - // 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp - static mlir::Block *getBlockForAllocaInsert(mlir::Operation *op) { - if (auto iface = - mlir::dyn_cast(op)) - return iface.getAllocaBlock(); - if (auto llvmFuncOp = mlir::dyn_cast(op)) - return &llvmFuncOp.front(); - return getBlockForAllocaInsert(op->getParentOp()); - } - - // Generate an alloca of size 1 for an object of type \p llvmObjectTy in the - // allocation address space provided for the architecture in the DataLayout - // specification. If the address space is different from the devices - // program address space we perform a cast. In the case of most architectures - // the program and allocation address space will be the default of 0 and no - // cast will be emitted. - mlir::Value genAllocaAndAddrCastWithType( - mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment, - mlir::ConversionPatternRewriter &rewriter) const { - auto thisPt = rewriter.saveInsertionPoint(); - mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); - if (mlir::isa(parentOp)) { - // DeclareReductionOp has multiple child regions. We want to get the first - // block of whichever of those regions we are currently in - mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent(); - rewriter.setInsertionPointToStart(&parentRegion->front()); - } else { - mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp); - rewriter.setInsertionPointToStart(insertBlock); - } - auto size = genI32Constant(loc, rewriter, 1); - unsigned allocaAs = getAllocaAddressSpace(rewriter); - unsigned programAs = getProgramAddressSpace(rewriter); - - mlir::Value al = rewriter.create( - loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), - llvmObjectTy, size, alignment); - - // if our allocation address space, is not the same as the program address - // space, then we must emit a cast to the program address space before use. - // An example case would be on AMDGPU, where the allocation address space is - // the numeric value 5 (private), and the program address space is 0 - // (generic). - if (allocaAs != programAs) { - al = rewriter.create( - loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al); - } - - rewriter.restoreInsertionPoint(thisPt); - return al; - } - - const fir::LLVMTypeConverter &lowerTy() const { - return *static_cast( - this->getTypeConverter()); - } - - void attachTBAATag(mlir::LLVM::AliasAnalysisOpInterface op, - mlir::Type baseFIRType, mlir::Type accessFIRType, - mlir::LLVM::GEPOp gep) const { - lowerTy().attachTBAATag(op, baseFIRType, accessFIRType, gep); - } - - const fir::FIRToLLVMPassOptions &options; -}; - -/// FIR conversion pattern template -template -class FIROpAndTypeConversion : public FIROpConversion { -public: - using FIROpConversion::FIROpConversion; - using OpAdaptor = typename FromOp::Adaptor; - - mlir::LogicalResult - matchAndRewrite(FromOp op, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const final { - mlir::Type ty = this->convertType(op.getType()); - return doRewrite(op, ty, adaptor, rewriter); - } - - virtual mlir::LogicalResult - doRewrite(FromOp addr, mlir::Type ty, OpAdaptor adaptor, - mlir::ConversionPatternRewriter &rewriter) const = 0; -}; -} // namespace - namespace { /// Lower `fir.address_of` operation to `llvm.address_of` operation. -struct AddrOfOpConversion : public FIROpConversion { +struct AddrOfOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -539,7 +189,7 @@ genAllocationScaleSize(OP op, mlir::Type ity, namespace { /// convert to LLVM IR dialect `alloca` -struct AllocaOpConversion : public FIROpConversion { +struct AllocaOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -624,7 +274,7 @@ struct AllocaOpConversion : public FIROpConversion { namespace { /// Lower `fir.box_addr` to the sequence of operations to extract the first /// element of the box. -struct BoxAddrOpConversion : public FIROpConversion { +struct BoxAddrOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -645,7 +295,7 @@ struct BoxAddrOpConversion : public FIROpConversion { /// Convert `!fir.boxchar_len` to `!llvm.extractvalue` for the 2nd part of the /// boxchar. -struct BoxCharLenOpConversion : public FIROpConversion { +struct BoxCharLenOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -668,7 +318,7 @@ struct BoxCharLenOpConversion : public FIROpConversion { /// Lower `fir.box_dims` to a sequence of operations to extract the requested /// dimension information from the boxed value. /// Result in a triple set of GEPs and loads. -struct BoxDimsOpConversion : public FIROpConversion { +struct BoxDimsOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -690,7 +340,7 @@ struct BoxDimsOpConversion : public FIROpConversion { /// Lower `fir.box_elesize` to a sequence of operations ro extract the size of /// an element in the boxed value. -struct BoxEleSizeOpConversion : public FIROpConversion { +struct BoxEleSizeOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -708,7 +358,7 @@ struct BoxEleSizeOpConversion : public FIROpConversion { /// Lower `fir.box_isalloc` to a sequence of operations to determine if the /// boxed value was from an ALLOCATABLE entity. -struct BoxIsAllocOpConversion : public FIROpConversion { +struct BoxIsAllocOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -726,7 +376,7 @@ struct BoxIsAllocOpConversion : public FIROpConversion { /// Lower `fir.box_isarray` to a sequence of operations to determine if the /// boxed is an array. -struct BoxIsArrayOpConversion : public FIROpConversion { +struct BoxIsArrayOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -746,7 +396,7 @@ struct BoxIsArrayOpConversion : public FIROpConversion { /// Lower `fir.box_isptr` to a sequence of operations to determined if the /// boxed value was from a POINTER entity. -struct BoxIsPtrOpConversion : public FIROpConversion { +struct BoxIsPtrOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -764,7 +414,7 @@ struct BoxIsPtrOpConversion : public FIROpConversion { /// Lower `fir.box_rank` to the sequence of operation to extract the rank from /// the box. -struct BoxRankOpConversion : public FIROpConversion { +struct BoxRankOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -784,7 +434,8 @@ struct BoxRankOpConversion : public FIROpConversion { /// Lower `fir.boxproc_host` operation. Extracts the host pointer from the /// boxproc. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct BoxProcHostOpConversion : public FIROpConversion { +struct BoxProcHostOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -797,7 +448,8 @@ struct BoxProcHostOpConversion : public FIROpConversion { /// Lower `fir.box_tdesc` to the sequence of operations to extract the type /// descriptor from the box. -struct BoxTypeDescOpConversion : public FIROpConversion { +struct BoxTypeDescOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -814,7 +466,8 @@ struct BoxTypeDescOpConversion : public FIROpConversion { /// Lower `fir.box_typecode` to a sequence of operations to extract the type /// code in the boxed value. -struct BoxTypeCodeOpConversion : public FIROpConversion { +struct BoxTypeCodeOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -832,7 +485,7 @@ struct BoxTypeCodeOpConversion : public FIROpConversion { }; /// Lower `fir.string_lit` to LLVM IR dialect operation. -struct StringLitOpConversion : public FIROpConversion { +struct StringLitOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -872,7 +525,7 @@ struct StringLitOpConversion : public FIROpConversion { }; /// `fir.call` -> `llvm.call` -struct CallOpConversion : public FIROpConversion { +struct CallOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -903,7 +556,7 @@ namespace { /// Per 10.1, the only comparisons available are .EQ. (oeq) and .NE. (une). /// /// For completeness, all other comparison are done on the real component only. -struct CmpcOpConversion : public FIROpConversion { +struct CmpcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -941,7 +594,7 @@ struct CmpcOpConversion : public FIROpConversion { }; /// Lower complex constants -struct ConstcOpConversion : public FIROpConversion { +struct ConstcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -968,7 +621,7 @@ struct ConstcOpConversion : public FIROpConversion { }; /// convert value of from-type to value of to-type -struct ConvertOpConversion : public FIROpConversion { +struct ConvertOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; static bool isFloatingPointTy(mlir::Type ty) { @@ -1137,7 +790,7 @@ struct ConvertOpConversion : public FIROpConversion { /// only used to carry information during FIR to FIR passes. It may be used /// in the future to generate the runtime type info data structures instead /// of generating them in lowering. -struct TypeInfoOpConversion : public FIROpConversion { +struct TypeInfoOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1150,7 +803,7 @@ struct TypeInfoOpConversion : public FIROpConversion { /// `fir.dt_entry` operation has no specific CodeGen. The operation is only used /// to carry information during FIR to FIR passes. -struct DTEntryOpConversion : public FIROpConversion { +struct DTEntryOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1162,7 +815,7 @@ struct DTEntryOpConversion : public FIROpConversion { }; /// Lower `fir.global_len` operation. -struct GlobalLenOpConversion : public FIROpConversion { +struct GlobalLenOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1175,7 +828,7 @@ struct GlobalLenOpConversion : public FIROpConversion { /// Lower fir.len_param_index struct LenParamIndexOpConversion - : public FIROpConversion { + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; // FIXME: this should be specialized by the runtime target @@ -1190,7 +843,7 @@ struct LenParamIndexOpConversion /// instructions that generate `!llvm.struct<(ptr, i64)>`. The 1st element /// in this struct is a pointer. Its type is determined from `KIND`. The 2nd /// element is the length of the character buffer (`#n`). -struct EmboxCharOpConversion : public FIROpConversion { +struct EmboxCharOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1283,7 +936,7 @@ genTypeStrideInBytes(mlir::Location loc, mlir::Type idxTy, namespace { /// Lower a `fir.allocmem` instruction into `llvm.call @malloc` -struct AllocMemOpConversion : public FIROpConversion { +struct AllocMemOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1352,7 +1005,7 @@ static unsigned getDimension(mlir::LLVM::LLVMArrayType ty) { namespace { /// Lower a `fir.freemem` instruction into `llvm.call @free` -struct FreeMemOpConversion : public FIROpConversion { +struct FreeMemOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -1410,9 +1063,9 @@ convertSubcomponentIndices(mlir::Location loc, mlir::Type eleTy, /// Common base class for embox to descriptor conversion. template -struct EmboxCommonConversion : public FIROpConversion { - using FIROpConversion::FIROpConversion; - using TypePair = typename FIROpConversion::TypePair; +struct EmboxCommonConversion : public fir::FIROpConversion { + using fir::FIROpConversion::FIROpConversion; + using TypePair = typename fir::FIROpConversion::TypePair; static int getCFIAttr(fir::BaseBoxType boxTy) { auto eleTy = boxTy.getEleTy(); @@ -1434,8 +1087,8 @@ struct EmboxCommonConversion : public FIROpConversion { return size; // Length accounted for in the genTypeStrideInBytes GEP. // Otherwise, multiply the single character size by the length. assert(!lenParams.empty()); - auto len64 = FIROpConversion::integerCast(loc, rewriter, i64Ty, - lenParams.back()); + auto len64 = fir::FIROpConversion::integerCast(loc, rewriter, i64Ty, + lenParams.back()); return rewriter.create(loc, i64Ty, size, len64); } @@ -2280,7 +1933,7 @@ struct XReboxOpConversion : public EmboxCommonConversion { /// Lower `fir.emboxproc` operation. Creates a procedure box. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct EmboxProcOpConversion : public FIROpConversion { +struct EmboxProcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2346,7 +1999,7 @@ struct ValueOpCommon { namespace { /// Extract a subobject value from an ssa-value of aggregate type struct ExtractValueOpConversion - : public FIROpAndTypeConversion, + : public fir::FIROpAndTypeConversion, public ValueOpCommon { using FIROpAndTypeConversion::FIROpAndTypeConversion; @@ -2365,7 +2018,7 @@ struct ExtractValueOpConversion /// InsertValue is the generalized instruction for the composition of new /// aggregate type values. struct InsertValueOpConversion - : public FIROpAndTypeConversion, + : public fir::FIROpAndTypeConversion, public ValueOpCommon { using FIROpAndTypeConversion::FIROpAndTypeConversion; @@ -2383,7 +2036,7 @@ struct InsertValueOpConversion /// InsertOnRange inserts a value into a sequence over a range of offsets. struct InsertOnRangeOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; // Increments an array of subscripts in a row major fasion. @@ -2447,7 +2100,7 @@ namespace { /// (See the static restriction on coordinate_of.) array_coor determines the /// coordinate (location) of a specific element. struct XArrayCoorOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; mlir::LogicalResult @@ -2615,7 +2268,7 @@ struct XArrayCoorOpConversion /// With unboxed arrays, there is the restriction that the array have a static /// shape in all but the last column. struct CoordinateOpConversion - : public FIROpAndTypeConversion { + : public fir::FIROpAndTypeConversion { using FIROpAndTypeConversion::FIROpAndTypeConversion; mlir::LogicalResult @@ -2910,7 +2563,7 @@ struct CoordinateOpConversion /// Convert `fir.field_index`. The conversion depends on whether the size of /// the record is static or dynamic. -struct FieldIndexOpConversion : public FIROpConversion { +struct FieldIndexOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; // NB: most field references should be resolved by this point @@ -2951,7 +2604,7 @@ struct FieldIndexOpConversion : public FIROpConversion { }; /// Convert `fir.end` -struct FirEndOpConversion : public FIROpConversion { +struct FirEndOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2963,7 +2616,7 @@ struct FirEndOpConversion : public FIROpConversion { }; /// Lower `fir.type_desc` to a global addr. -struct TypeDescOpConversion : public FIROpConversion { +struct TypeDescOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -2990,7 +2643,7 @@ struct TypeDescOpConversion : public FIROpConversion { }; /// Lower `fir.has_value` operation to `llvm.return` operation. -struct HasValueOpConversion : public FIROpConversion { +struct HasValueOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3043,7 +2696,7 @@ static inline bool attributeTypeIsCompatible(mlir::MLIRContext *ctx, /// Lower `fir.global` operation to `llvm.global` operation. /// `fir.insert_on_range` operations are replaced with constant dense attribute /// if they are applied on the full range. -struct GlobalOpConversion : public FIROpConversion { +struct GlobalOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3167,7 +2820,7 @@ struct GlobalOpConversion : public FIROpConversion { }; /// `fir.load` --> `llvm.load` -struct LoadOpConversion : public FIROpConversion { +struct LoadOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3216,7 +2869,7 @@ struct LoadOpConversion : public FIROpConversion { /// Lower `fir.no_reassoc` to LLVM IR dialect. /// TODO: how do we want to enforce this in LLVM-IR? Can we manipulate the fast /// math flags? -struct NoReassocOpConversion : public FIROpConversion { +struct NoReassocOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3277,7 +2930,7 @@ static void genCaseLadderStep(mlir::Location loc, mlir::Value cmp, /// upper bound in the same case condition. /// /// TODO: lowering of CHARACTER type cases is not handled yet. -struct SelectCaseOpConversion : public FIROpConversion { +struct SelectCaseOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3391,7 +3044,7 @@ static void selectMatchAndRewrite(const fir::LLVMTypeConverter &lowering, } /// conversion of fir::SelectOp to an if-then-else ladder -struct SelectOpConversion : public FIROpConversion { +struct SelectOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3403,7 +3056,7 @@ struct SelectOpConversion : public FIROpConversion { }; /// conversion of fir::SelectRankOp to an if-then-else ladder -struct SelectRankOpConversion : public FIROpConversion { +struct SelectRankOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3415,7 +3068,7 @@ struct SelectRankOpConversion : public FIROpConversion { }; /// Lower `fir.select_type` to LLVM IR dialect. -struct SelectTypeOpConversion : public FIROpConversion { +struct SelectTypeOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3428,7 +3081,7 @@ struct SelectTypeOpConversion : public FIROpConversion { }; /// `fir.store` --> `llvm.store` -struct StoreOpConversion : public FIROpConversion { +struct StoreOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3462,7 +3115,7 @@ namespace { /// Convert `fir.unboxchar` into two `llvm.extractvalue` instructions. One for /// the character buffer and one for the buffer length. -struct UnboxCharOpConversion : public FIROpConversion { +struct UnboxCharOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3487,7 +3140,7 @@ struct UnboxCharOpConversion : public FIROpConversion { /// Lower `fir.unboxproc` operation. Unbox a procedure box value, yielding its /// components. /// TODO: Part of supporting Fortran 2003 procedure pointers. -struct UnboxProcOpConversion : public FIROpConversion { +struct UnboxProcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3499,7 +3152,7 @@ struct UnboxProcOpConversion : public FIROpConversion { }; /// convert to LLVM IR dialect `undef` -struct UndefOpConversion : public FIROpConversion { +struct UndefOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3511,7 +3164,7 @@ struct UndefOpConversion : public FIROpConversion { } }; -struct ZeroOpConversion : public FIROpConversion { +struct ZeroOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3524,7 +3177,8 @@ struct ZeroOpConversion : public FIROpConversion { }; /// `fir.unreachable` --> `llvm.unreachable` -struct UnreachableOpConversion : public FIROpConversion { +struct UnreachableOpConversion + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3541,7 +3195,7 @@ struct UnreachableOpConversion : public FIROpConversion { /// %1 = llvm.ptrtoint %0 /// %2 = llvm.icmp "ne" %1, %0 : i64 /// ``` -struct IsPresentOpConversion : public FIROpConversion { +struct IsPresentOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3570,7 +3224,7 @@ struct IsPresentOpConversion : public FIROpConversion { /// Create value signaling an absent optional argument in a call, e.g. /// `fir.absent !fir.ref` --> `llvm.mlir.zero : !llvm.ptr` -struct AbsentOpConversion : public FIROpConversion { +struct AbsentOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3630,7 +3284,7 @@ complexSum(OPTY sumop, mlir::ValueRange opnds, } // namespace namespace { -struct AddcOpConversion : public FIROpConversion { +struct AddcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3645,7 +3299,7 @@ struct AddcOpConversion : public FIROpConversion { } }; -struct SubcOpConversion : public FIROpConversion { +struct SubcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3661,7 +3315,7 @@ struct SubcOpConversion : public FIROpConversion { }; /// Inlined complex multiply -struct MulcOpConversion : public FIROpConversion { +struct MulcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3695,7 +3349,7 @@ struct MulcOpConversion : public FIROpConversion { }; /// Inlined complex division -struct DivcOpConversion : public FIROpConversion { +struct DivcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3735,7 +3389,7 @@ struct DivcOpConversion : public FIROpConversion { }; /// Inlined complex negation -struct NegcOpConversion : public FIROpConversion { +struct NegcOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3756,7 +3410,7 @@ struct NegcOpConversion : public FIROpConversion { } }; -struct BoxOffsetOpConversion : public FIROpConversion { +struct BoxOffsetOpConversion : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult @@ -3782,10 +3436,10 @@ struct BoxOffsetOpConversion : public FIROpConversion { /// anymore uses. /// These operations are normally dead after the pre-codegen pass. template -struct MustBeDeadConversion : public FIROpConversion { +struct MustBeDeadConversion : public fir::FIROpConversion { explicit MustBeDeadConversion(const fir::LLVMTypeConverter &lowering, const fir::FIRToLLVMPassOptions &options) - : FIROpConversion(lowering, options) {} + : fir::FIROpConversion(lowering, options) {} using OpAdaptor = typename FromOp::Adaptor; mlir::LogicalResult @@ -3799,7 +3453,7 @@ struct MustBeDeadConversion : public FIROpConversion { }; struct UnrealizedConversionCastOpConversion - : public FIROpConversion { + : public fir::FIROpConversion { using FIROpConversion::FIROpConversion; mlir::LogicalResult diff --git a/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp new file mode 100644 index 0000000000000..26871d8888155 --- /dev/null +++ b/flang/lib/Optimizer/CodeGen/FIROpPatterns.cpp @@ -0,0 +1,315 @@ +//===-- CodeGen.cpp -- bridge to lower to LLVM ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ +// +//===----------------------------------------------------------------------===// + +#include "flang/Optimizer/CodeGen/FIROpPatterns.h" +#include "mlir/Dialect/OpenMP/OpenMPDialect.h" +#include "llvm/Support/Debug.h" + +static inline mlir::Type getLlvmPtrType(mlir::MLIRContext *context, + unsigned addressSpace = 0) { + return mlir::LLVM::LLVMPointerType::get(context, addressSpace); +} + +static unsigned getTypeDescFieldId(mlir::Type ty) { + auto isArray = fir::dyn_cast_ptrOrBoxEleTy(ty).isa(); + return isArray ? kOptTypePtrPosInBox : kDimsPosInBox; +} + +namespace fir { + +ConvertFIRToLLVMPattern::ConvertFIRToLLVMPattern( + llvm::StringRef rootOpName, mlir::MLIRContext *context, + const fir::LLVMTypeConverter &typeConverter, + const fir::FIRToLLVMPassOptions &options, mlir::PatternBenefit benefit) + : ConvertToLLVMPattern(rootOpName, context, typeConverter, benefit), + options(options) {} + +// Convert FIR type to LLVM without turning fir.box into memory +// reference. +mlir::Type +ConvertFIRToLLVMPattern::convertObjectType(mlir::Type firType) const { + if (auto boxTy = firType.dyn_cast()) + return lowerTy().convertBoxTypeAsStruct(boxTy); + return lowerTy().convertType(firType); +} + +mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genI32Constant( + mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int value) const { + mlir::Type i32Ty = rewriter.getI32Type(); + mlir::IntegerAttr attr = rewriter.getI32IntegerAttr(value); + return rewriter.create(loc, i32Ty, attr); +} + +mlir::LLVM::ConstantOp ConvertFIRToLLVMPattern::genConstantOffset( + mlir::Location loc, mlir::ConversionPatternRewriter &rewriter, + int offset) const { + mlir::Type ity = lowerTy().offsetType(); + mlir::IntegerAttr cattr = rewriter.getI32IntegerAttr(offset); + return rewriter.create(loc, ity, cattr); +} + +/// Perform an extension or truncation as needed on an integer value. Lowering +/// to the specific target may involve some sign-extending or truncation of +/// values, particularly to fit them from abstract box types to the +/// appropriate reified structures. +mlir::Value +ConvertFIRToLLVMPattern::integerCast(mlir::Location loc, + mlir::ConversionPatternRewriter &rewriter, + mlir::Type ty, mlir::Value val) const { + auto valTy = val.getType(); + // If the value was not yet lowered, lower its type so that it can + // be used in getPrimitiveTypeSizeInBits. + if (!valTy.isa()) + valTy = convertType(valTy); + auto toSize = mlir::LLVM::getPrimitiveTypeSizeInBits(ty); + auto fromSize = mlir::LLVM::getPrimitiveTypeSizeInBits(valTy); + if (toSize < fromSize) + return rewriter.create(loc, ty, val); + if (toSize > fromSize) + return rewriter.create(loc, ty, val); + return val; +} + +fir::ConvertFIRToLLVMPattern::TypePair +ConvertFIRToLLVMPattern::getBoxTypePair(mlir::Type firBoxTy) const { + mlir::Type llvmBoxTy = + lowerTy().convertBoxTypeAsStruct(mlir::cast(firBoxTy)); + return TypePair{firBoxTy, llvmBoxTy}; +} + +/// Construct code sequence to extract the specific value from a `fir.box`. +mlir::Value ConvertFIRToLLVMPattern::getValueFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Type resultTy, + mlir::ConversionPatternRewriter &rewriter, int boxValue) const { + if (box.getType().isa()) { + auto pty = getLlvmPtrType(resultTy.getContext()); + auto p = rewriter.create( + loc, pty, boxTy.llvm, box, + llvm::ArrayRef{0, boxValue}); + auto loadOp = rewriter.create(loc, resultTy, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; + } + return rewriter.create(loc, box, boxValue); +} + +/// Method to construct code sequence to get the triple for dimension `dim` +/// from a box. +llvm::SmallVector ConvertFIRToLLVMPattern::getDimsFromBox( + mlir::Location loc, llvm::ArrayRef retTys, TypePair boxTy, + mlir::Value box, mlir::Value dim, + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Value l0 = + loadDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); + mlir::Value l1 = + loadDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); + mlir::Value l2 = + loadDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); + return {l0, l1, l2}; +} + +llvm::SmallVector ConvertFIRToLLVMPattern::getDimsFromBox( + mlir::Location loc, llvm::ArrayRef retTys, TypePair boxTy, + mlir::Value box, int dim, mlir::ConversionPatternRewriter &rewriter) const { + mlir::Value l0 = + getDimFieldFromBox(loc, boxTy, box, dim, 0, retTys[0], rewriter); + mlir::Value l1 = + getDimFieldFromBox(loc, boxTy, box, dim, 1, retTys[1], rewriter); + mlir::Value l2 = + getDimFieldFromBox(loc, boxTy, box, dim, 2, retTys[2], rewriter); + return {l0, l1, l2}; +} + +mlir::Value ConvertFIRToLLVMPattern::loadDimFieldFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, mlir::Value dim, + int off, mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const { + assert(box.getType().isa() && + "descriptor inquiry with runtime dim can only be done on descriptor " + "in memory"); + mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, + static_cast(kDimsPosInBox), dim, off); + auto loadOp = rewriter.create(loc, ty, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; +} + +mlir::Value ConvertFIRToLLVMPattern::getDimFieldFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, int dim, int off, + mlir::Type ty, mlir::ConversionPatternRewriter &rewriter) const { + if (box.getType().isa()) { + mlir::LLVM::GEPOp p = genGEP(loc, boxTy.llvm, rewriter, box, 0, + static_cast(kDimsPosInBox), dim, off); + auto loadOp = rewriter.create(loc, ty, p); + attachTBAATag(loadOp, boxTy.fir, nullptr, p); + return loadOp; + } + return rewriter.create( + loc, box, llvm::ArrayRef{kDimsPosInBox, dim, off}); +} + +mlir::Value ConvertFIRToLLVMPattern::getStrideFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, unsigned dim, + mlir::ConversionPatternRewriter &rewriter) const { + auto idxTy = lowerTy().indexType(); + return getDimFieldFromBox(loc, boxTy, box, dim, kDimStridePos, idxTy, + rewriter); +} + +/// Read base address from a fir.box. Returned address has type ty. +mlir::Value ConvertFIRToLLVMPattern::getBaseAddrFromBox( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Type resultTy = ::getLlvmPtrType(boxTy.llvm.getContext()); + return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kAddrPosInBox); +} + +mlir::Value ConvertFIRToLLVMPattern::getElementSizeFromBox( + mlir::Location loc, mlir::Type resultTy, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + return getValueFromBox(loc, boxTy, box, resultTy, rewriter, kElemLenPosInBox); +} + +// Get the element type given an LLVM type that is of the form +// (array|struct|vector)+ and the provided indexes. +mlir::Type ConvertFIRToLLVMPattern::getBoxEleTy( + mlir::Type type, llvm::ArrayRef indexes) const { + for (unsigned i : indexes) { + if (auto t = type.dyn_cast()) { + assert(!t.isOpaque() && i < t.getBody().size()); + type = t.getBody()[i]; + } else if (auto t = type.dyn_cast()) { + type = t.getElementType(); + } else if (auto t = type.dyn_cast()) { + type = t.getElementType(); + } else { + fir::emitFatalError(mlir::UnknownLoc::get(type.getContext()), + "request for invalid box element type"); + } + } + return type; +} + +// Return LLVM type of the object described by a fir.box of \p boxType. +mlir::Type ConvertFIRToLLVMPattern::getLlvmObjectTypeFromBoxType( + mlir::Type boxType) const { + mlir::Type objectType = fir::dyn_cast_ptrOrBoxEleTy(boxType); + assert(objectType && "boxType must be a box type"); + return this->convertType(objectType); +} + +/// Read the address of the type descriptor from a box. +mlir::Value ConvertFIRToLLVMPattern::loadTypeDescAddress( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter) const { + unsigned typeDescFieldId = getTypeDescFieldId(boxTy.fir); + mlir::Type tdescType = lowerTy().convertTypeDescType(rewriter.getContext()); + return getValueFromBox(loc, boxTy, box, tdescType, rewriter, typeDescFieldId); +} + +// Load the attribute from the \p box and perform a check against \p maskValue +// The final comparison is implemented as `(attribute & maskValue) != 0`. +mlir::Value ConvertFIRToLLVMPattern::genBoxAttributeCheck( + mlir::Location loc, TypePair boxTy, mlir::Value box, + mlir::ConversionPatternRewriter &rewriter, unsigned maskValue) const { + mlir::Type attrTy = rewriter.getI32Type(); + mlir::Value attribute = + getValueFromBox(loc, boxTy, box, attrTy, rewriter, kAttributePosInBox); + mlir::LLVM::ConstantOp attrMask = genConstantOffset(loc, rewriter, maskValue); + auto maskRes = + rewriter.create(loc, attrTy, attribute, attrMask); + mlir::LLVM::ConstantOp c0 = genConstantOffset(loc, rewriter, 0); + return rewriter.create(loc, mlir::LLVM::ICmpPredicate::ne, + maskRes, c0); +} + +// Find the Block in which the alloca should be inserted. +// The order to recursively find the proper block: +// 1. An OpenMP Op that will be outlined. +// 2. A LLVMFuncOp +// 3. The first ancestor that is an OpenMP Op or a LLVMFuncOp +mlir::Block * +ConvertFIRToLLVMPattern::getBlockForAllocaInsert(mlir::Operation *op) const { + if (auto iface = mlir::dyn_cast(op)) + return iface.getAllocaBlock(); + if (auto llvmFuncOp = mlir::dyn_cast(op)) + return &llvmFuncOp.front(); + return getBlockForAllocaInsert(op->getParentOp()); +} + +// Generate an alloca of size 1 for an object of type \p llvmObjectTy in the +// allocation address space provided for the architecture in the DataLayout +// specification. If the address space is different from the devices +// program address space we perform a cast. In the case of most architectures +// the program and allocation address space will be the default of 0 and no +// cast will be emitted. +mlir::Value ConvertFIRToLLVMPattern::genAllocaAndAddrCastWithType( + mlir::Location loc, mlir::Type llvmObjectTy, unsigned alignment, + mlir::ConversionPatternRewriter &rewriter) const { + auto thisPt = rewriter.saveInsertionPoint(); + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + if (mlir::isa(parentOp)) { + // DeclareReductionOp has multiple child regions. We want to get the first + // block of whichever of those regions we are currently in + mlir::Region *parentRegion = rewriter.getInsertionBlock()->getParent(); + rewriter.setInsertionPointToStart(&parentRegion->front()); + } else { + mlir::Block *insertBlock = getBlockForAllocaInsert(parentOp); + rewriter.setInsertionPointToStart(insertBlock); + } + auto size = genI32Constant(loc, rewriter, 1); + unsigned allocaAs = getAllocaAddressSpace(rewriter); + unsigned programAs = getProgramAddressSpace(rewriter); + + mlir::Value al = rewriter.create( + loc, ::getLlvmPtrType(llvmObjectTy.getContext(), allocaAs), llvmObjectTy, + size, alignment); + + // if our allocation address space, is not the same as the program address + // space, then we must emit a cast to the program address space before use. + // An example case would be on AMDGPU, where the allocation address space is + // the numeric value 5 (private), and the program address space is 0 + // (generic). + if (allocaAs != programAs) { + al = rewriter.create( + loc, ::getLlvmPtrType(llvmObjectTy.getContext(), programAs), al); + } + + rewriter.restoreInsertionPoint(thisPt); + return al; +} + +unsigned ConvertFIRToLLVMPattern::getAllocaAddressSpace( + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + assert(parentOp != nullptr && + "expected insertion block to have parent operation"); + if (auto module = parentOp->getParentOfType()) + if (mlir::Attribute addrSpace = + mlir::DataLayout(module).getAllocaMemorySpace()) + return llvm::cast(addrSpace).getUInt(); + return defaultAddressSpace; +} + +unsigned ConvertFIRToLLVMPattern::getProgramAddressSpace( + mlir::ConversionPatternRewriter &rewriter) const { + mlir::Operation *parentOp = rewriter.getInsertionBlock()->getParentOp(); + assert(parentOp != nullptr && + "expected insertion block to have parent operation"); + if (auto module = parentOp->getParentOfType()) + if (mlir::Attribute addrSpace = + mlir::DataLayout(module).getProgramMemorySpace()) + return llvm::cast(addrSpace).getUInt(); + return defaultAddressSpace; +} + +} // namespace fir