diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h index d4b223863c5490..be6c8cd084ecad 100644 --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -197,6 +197,10 @@ class TargetLibraryInfoImpl { /// Returns the size of the wchar_t type in bytes or 0 if the size is unknown. /// This queries the 'wchar_size' metadata. unsigned getWCharSize(const Module &M) const; + + /// Returns the largest vectorization factor used in the list of + /// vector functions. + unsigned getWidestVF(StringRef ScalarF) const; }; /// Provides information about what library functions are available for @@ -337,6 +341,12 @@ class TargetLibraryInfo { FunctionAnalysisManager::Invalidator &) { return false; } + + /// Returns the largest vectorization factor used in the list of + /// vector functions. + unsigned getWidestVF(StringRef ScalarF) const { + return Impl->getWidestVF(ScalarF); + } }; /// Analysis pass providing the \c TargetLibraryInfo. diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index 034d851abcc828..dbae32e843936f 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -180,6 +180,7 @@ void initializeIndVarSimplifyLegacyPassPass(PassRegistry&); void initializeIndirectBrExpandPassPass(PassRegistry&); void initializeInferAddressSpacesPass(PassRegistry&); void initializeInferFunctionAttrsLegacyPassPass(PassRegistry&); +void initializeInjectTLIMappingsLegacyPass(PassRegistry &); void initializeInlineCostAnalysisPass(PassRegistry&); void initializeInstCountPass(PassRegistry&); void initializeInstNamerPass(PassRegistry&); diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h index 46d8f0c8ed8192..1454f3b932bdb0 100644 --- a/llvm/include/llvm/LinkAllPasses.h +++ b/llvm/include/llvm/LinkAllPasses.h @@ -225,6 +225,7 @@ namespace { (void) llvm::createScalarizeMaskedMemIntrinPass(); (void) llvm::createWarnMissedTransformationsPass(); (void) llvm::createHardwareLoopsPass(); + (void)llvm::createInjectTLIMappingsLegacyPass(); (void)new llvm::IntervalPartition(); (void)new llvm::ScalarEvolutionWrapperPass(); diff --git a/llvm/include/llvm/Transforms/Utils.h b/llvm/include/llvm/Transforms/Utils.h index 6e03453babf1a3..bb31646ce46226 100644 --- a/llvm/include/llvm/Transforms/Utils.h +++ b/llvm/include/llvm/Transforms/Utils.h @@ -119,6 +119,13 @@ ModulePass *createStripNonLineTableDebugInfoPass(); // number of conditional branches in the hot paths based on profiles. // FunctionPass *createControlHeightReductionLegacyPass(); + +//===----------------------------------------------------------------------===// +// +// InjectTLIMappingsLegacy - populates the VFABI attribute with the +// scalar-to-vector mappings from the TargetLibraryInfo. +// +FunctionPass *createInjectTLIMappingsLegacyPass(); } #endif diff --git a/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h new file mode 100644 index 00000000000000..84e4fee51c2639 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/InjectTLIMappings.h @@ -0,0 +1,37 @@ +//===- InjectTLIMAppings.h - TLI to VFABI attribute injection ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Populates the VFABI attribute with the scalar-to-vector mappings +// from the TargetLibraryInfo. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H +#define LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H + +#include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" + +namespace llvm { +class InjectTLIMappings : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +// Legacy pass +class InjectTLIMappingsLegacy : public FunctionPass { +public: + static char ID; + InjectTLIMappingsLegacy() : FunctionPass(ID) { + initializeInjectTLIMappingsLegacyPass(*PassRegistry::getPassRegistry()); + } + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnFunction(Function &F) override; +}; + +} // End namespace llvm +#endif // LLVM_TRANSFORMS_UTILS_INJECTTLIMAPPINGS_H diff --git a/llvm/lib/Analysis/TargetLibraryInfo.cpp b/llvm/lib/Analysis/TargetLibraryInfo.cpp index 0d0e8f1eecaf64..f1d4268ad42649 100644 --- a/llvm/lib/Analysis/TargetLibraryInfo.cpp +++ b/llvm/lib/Analysis/TargetLibraryInfo.cpp @@ -1637,3 +1637,19 @@ INITIALIZE_PASS(TargetLibraryInfoWrapperPass, "targetlibinfo", char TargetLibraryInfoWrapperPass::ID = 0; void TargetLibraryInfoWrapperPass::anchor() {} + +unsigned TargetLibraryInfoImpl::getWidestVF(StringRef ScalarF) const { + ScalarF = sanitizeFunctionName(ScalarF); + if (ScalarF.empty()) + return 1; + + unsigned VF = 1; + std::vector::const_iterator I = + llvm::lower_bound(VectorDescs, ScalarF, compareWithScalarFnName); + while (I != VectorDescs.end() && StringRef(I->ScalarFnName) == ScalarF) { + if (I->VectorizationFactor > VF) + VF = I->VectorizationFactor; + ++I; + } + return VF; +} diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index d11cb6a4d3cddf..44043bd582c6e0 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -1166,6 +1166,9 @@ void VFABI::getVectorVariantNames( const StringRef S = CI.getAttribute(AttributeList::FunctionIndex, VFABI::MappingsAttrName) .getValueAsString(); + if (S.empty()) + return; + SmallVector ListAttr; S.split(ListAttr, ","); diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index aa232b2c1c8a37..5896dbf5bb9852 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -170,6 +170,7 @@ #include "llvm/Transforms/Utils/BreakCriticalEdges.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" #include "llvm/Transforms/Utils/EntryExitInstrumenter.h" +#include "llvm/Transforms/Utils/InjectTLIMappings.h" #include "llvm/Transforms/Utils/LCSSA.h" #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h" #include "llvm/Transforms/Utils/LoopSimplify.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 1fa274d172b19a..d988506b5e9806 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -184,6 +184,7 @@ FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("float2int", Float2IntPass()) FUNCTION_PASS("no-op-function", NoOpFunctionPass()) FUNCTION_PASS("libcalls-shrinkwrap", LibCallsShrinkWrapPass()) +FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings()) FUNCTION_PASS("loweratomic", LowerAtomicPass()) FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass()) FUNCTION_PASS("lower-guard-intrinsic", LowerGuardIntrinsicPass()) diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt index b1d9e062903c40..dd2661aa23ffd6 100644 --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -23,6 +23,7 @@ add_llvm_library(LLVMTransformUtils GuardUtils.cpp InlineFunction.cpp ImportedFunctionsInliningStatistics.cpp + InjectTLIMappings.cpp InstructionNamer.cpp IntegerDivision.cpp LCSSA.cpp diff --git a/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp new file mode 100644 index 00000000000000..9192e74b9ace98 --- /dev/null +++ b/llvm/lib/Transforms/Utils/InjectTLIMappings.cpp @@ -0,0 +1,186 @@ +//===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Populates the VFABI attribute with the scalar-to-vector mappings +// from the TargetLibraryInfo. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/InjectTLIMappings.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/VectorUtils.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/Transforms/Utils.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" + +using namespace llvm; + +#define DEBUG_TYPE "inject-tli-mappings" + +STATISTIC(NumCallInjected, + "Number of calls in which the mappings have been injected."); + +STATISTIC(NumVFDeclAdded, + "Number of function declarations that have been added."); +STATISTIC(NumCompUsedAdded, + "Number of `@llvm.compiler.used` operands that have been added."); + +/// Helper function to map the TLI name to a strings that holds +/// scalar-to-vector mapping. +/// +/// _ZGV_() +/// +/// where: +/// +/// = "_LLVM_" +/// = "N". Note: TLI does not support masked interfaces. +/// = Number of concurrent lanes, stored in the `VectorizationFactor` +/// field of the `VecDesc` struct. +/// = "v", as many as are the number of parameters of CI. +/// = the name of the scalar function called by CI. +/// = the name of the vector function mapped by the TLI. +static std::string mangleTLIName(StringRef VectorName, const CallInst &CI, + unsigned VF) { + SmallString<256> Buffer; + llvm::raw_svector_ostream Out(Buffer); + Out << "_ZGV" << VFABI::_LLVM_ << "N" << VF; + for (unsigned I = 0; I < CI.getNumArgOperands(); ++I) + Out << "v"; + Out << "_" << CI.getCalledFunction()->getName() << "(" << VectorName << ")"; + return Out.str(); +} + +/// A helper function for converting Scalar types to vector types. +/// If the incoming type is void, we return void. If the VF is 1, we return +/// the scalar type. +static Type *ToVectorTy(Type *Scalar, unsigned VF, bool isScalable = false) { + if (Scalar->isVoidTy() || VF == 1) + return Scalar; + return VectorType::get(Scalar, {VF, isScalable}); +} + +/// A helper function that adds the vector function declaration that +/// vectorizes the CallInst CI with a vectorization factor of VF +/// lanes. The TLI assumes that all parameters and the return type of +/// CI (other than void) need to be widened to a VectorType of VF +/// lanes. +static void addVariantDeclaration(CallInst &CI, const unsigned VF, + const StringRef VFName) { + Module *M = CI.getModule(); + + // Add function declaration. + Type *RetTy = ToVectorTy(CI.getType(), VF); + SmallVector Tys; + for (Value *ArgOperand : CI.arg_operands()) + Tys.push_back(ToVectorTy(ArgOperand->getType(), VF)); + assert(!CI.getFunctionType()->isVarArg() && + "VarArg functions are not supported."); + FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false); + Function *VectorF = + Function::Create(FTy, Function::ExternalLinkage, VFName, M); + VectorF->copyAttributesFrom(CI.getCalledFunction()); + ++NumVFDeclAdded; + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName + << "` of type " << *(VectorF->getType()) << "\n"); + + // Make function declaration (without a body) "sticky" in the IR by + // listing it in the @llvm.compiler.used intrinsic. + assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` " + "only on declarations."); + appendToCompilerUsed(*M, {VectorF}); + LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName + << "` to `@llvm.compiler.used`.\n"); + ++NumCompUsedAdded; +} + +static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) { + // This is needed to make sure we don't query the TLI for calls to + // bitcast of function pointers, like `%call = call i32 (i32*, ...) + // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`, + // as such calls make the `isFunctionVectorizable` raise an + // exception. + if (CI.isNoBuiltin() || !CI.getCalledFunction()) + return; + + const std::string ScalarName = CI.getCalledFunction()->getName(); + // Nothing to be done if the TLI thinks the function is not + // vectorizable. + if (!TLI.isFunctionVectorizable(ScalarName)) + return; + SmallVector Mappings; + VFABI::getVectorVariantNames(CI, Mappings); + Module *M = CI.getModule(); + const SetVector OriginalSetOfMappings(Mappings.begin(), + Mappings.end()); + // All VFs in the TLI are powers of 2. + for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF; + VF *= 2) { + const std::string TLIName = TLI.getVectorizedFunction(ScalarName, VF); + if (!TLIName.empty()) { + std::string MangledName = mangleTLIName(TLIName, CI, VF); + if (!OriginalSetOfMappings.count(MangledName)) { + Mappings.push_back(MangledName); + ++NumCallInjected; + } + Function *VariantF = M->getFunction(TLIName); + if (!VariantF) + addVariantDeclaration(CI, VF, TLIName); + } + } + + VFABI::setVectorVariantNames(&CI, Mappings); +} + +static bool runImpl(const TargetLibraryInfo &TLI, Function &F) { + for (auto &I : instructions(F)) + if (auto CI = dyn_cast(&I)) + addMappingsFromTLI(TLI, *CI); + // Even if the pass adds IR attributes, the analyses are preserved. + return false; +} + +//////////////////////////////////////////////////////////////////////////////// +// New pass manager implementation. +//////////////////////////////////////////////////////////////////////////////// +PreservedAnalyses InjectTLIMappings::run(Function &F, + FunctionAnalysisManager &AM) { + const TargetLibraryInfo &TLI = AM.getResult(F); + runImpl(TLI, F); + // Even if the pass adds IR attributes, the analyses are preserved. + return PreservedAnalyses::all(); +} + +//////////////////////////////////////////////////////////////////////////////// +// Legacy PM Implementation. +//////////////////////////////////////////////////////////////////////////////// +bool InjectTLIMappingsLegacy::runOnFunction(Function &F) { + const TargetLibraryInfo &TLI = + getAnalysis().getTLI(F); + return runImpl(TLI, F); +} + +void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesCFG(); + AU.addRequired(); + AU.addPreserved(); +} + +//////////////////////////////////////////////////////////////////////////////// +// Legacy Pass manager initialization +//////////////////////////////////////////////////////////////////////////////// +char InjectTLIMappingsLegacy::ID = 0; + +INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE, + "Inject TLI Mappings", false, false) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings", + false, false) + +FunctionPass *llvm::createInjectTLIMappingsLegacyPass() { + return new InjectTLIMappingsLegacy(); +} diff --git a/llvm/lib/Transforms/Utils/Utils.cpp b/llvm/lib/Transforms/Utils/Utils.cpp index 5272ab6e95d50d..7769c7493cdab8 100644 --- a/llvm/lib/Transforms/Utils/Utils.cpp +++ b/llvm/lib/Transforms/Utils/Utils.cpp @@ -39,6 +39,7 @@ void llvm::initializeTransformUtils(PassRegistry &Registry) { initializeMetaRenamerPass(Registry); initializeStripGCRelocatesPass(Registry); initializePredicateInfoPrinterLegacyPassPass(Registry); + initializeInjectTLIMappingsLegacyPass(Registry); } /// LLVMInitializeTransformUtils - C binding for initializeTransformUtilsPasses. diff --git a/llvm/test/Transforms/Util/add-TLI-mappings.ll b/llvm/test/Transforms/Util/add-TLI-mappings.ll new file mode 100644 index 00000000000000..c68a9c9a71c65a --- /dev/null +++ b/llvm/test/Transforms/Util/add-TLI-mappings.ll @@ -0,0 +1,61 @@ +; RUN: opt -vector-library=SVML -inject-tli-mappings -S < %s | FileCheck %s --check-prefixes=COMMON,SVML +; RUN: opt -vector-library=SVML -passes=inject-tli-mappings -S < %s | FileCheck %s --check-prefixes=COMMON,SVML +; RUN: opt -vector-library=MASSV -inject-tli-mappings -S < %s | FileCheck %s --check-prefixes=COMMON,MASSV +; RUN: opt -vector-library=MASSV -passes=inject-tli-mappings -S < %s | FileCheck %s --check-prefixes=COMMON,MASSV +; RUN: opt -vector-library=Accelerate -inject-tli-mappings -S < %s | FileCheck %s --check-prefixes=COMMON,ACCELERATE +; RUN: opt -vector-library=Accelerate -passes=inject-tli-mappings -S < %s | FileCheck %s --check-prefixes=COMMON,ACCELERATE + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; COMMON-LABEL: @llvm.compiler.used = appending global +; SVML-SAME: [3 x i8*] [ +; SVML-SAME: i8* bitcast (<2 x double> (<2 x double>)* @__svml_sin2 to i8*), +; SVML-SAME: i8* bitcast (<4 x double> (<4 x double>)* @__svml_sin4 to i8*), +; SVML-SAME: i8* bitcast (<8 x double> (<8 x double>)* @__svml_sin8 to i8*) +; MASSV-SAME: [2 x i8*] [ +; MASSV-SAME: i8* bitcast (<2 x double> (<2 x double>)* @__sind2_massv to i8*), +; MASSV-SAME: i8* bitcast (<4 x float> (<4 x float>)* @__log10f4_massv to i8*) +; ACCELERATE-SAME: [1 x i8*] [ +; ACCELERATE-SAME: i8* bitcast (<4 x float> (<4 x float>)* @vlog10f to i8*) +; COMMON-SAME: ], section "llvm.metadata" + +define double @sin_f64(double %in) { +; COMMON-LABEL: @sin_f64( +; SVML: call double @sin(double %{{.*}}) #[[SIN:[0-9]+]] +; MASSV: call double @sin(double %{{.*}}) #[[SIN:[0-9]+]] +; ACCELERATE: call double @sin(double %{{.*}}) +; No mapping of "sin" to a vector function for Accelerate. +; ACCELERATE-NOT: _ZGV_LLVM_{{.*}}_sin({{.*}}) + %call = tail call double @sin(double %in) + ret double %call +} + +declare double @sin(double) #0 + +define float @call_llvm.log10.f32(float %in) { +; COMMON-LABEL: @call_llvm.log10.f32( +; SVML: call float @llvm.log10.f32(float %{{.*}}) +; MASSV: call float @llvm.log10.f32(float %{{.*}}) #[[LOG10:[0-9]+]] +; ACCELERATE: call float @llvm.log10.f32(float %{{.*}}) #[[LOG10:[0-9]+]] +; No mapping of "llvm.log10.f32" to a vector function for SVML. +; SVML-NOT: _ZGV_LLVM_{{.*}}_llvm.log10.f32({{.*}}) + %call = tail call float @llvm.log10.f32(float %in) + ret float %call +} + +declare float @llvm.log10.f32(float) #0 +attributes #0 = { nounwind readnone } + +; SVML: attributes #[[SIN]] = { "vector-function-abi-variant"= +; SVML-SAME: "_ZGV_LLVM_N2v_sin(__svml_sin2), +; SVML-SAME: _ZGV_LLVM_N4v_sin(__svml_sin4), +; SVML-SAME: _ZGV_LLVM_N8v_sin(__svml_sin8)" } + +; MASSV: attributes #[[SIN]] = { "vector-function-abi-variant"= +; MASSV-SAME: "_ZGV_LLVM_N2v_sin(__sind2_massv)" } +; MASSV: attributes #[[LOG10]] = { "vector-function-abi-variant"= +; MASSV-SAME: "_ZGV_LLVM_N4v_llvm.log10.f32(__log10f4_massv)" } + +; ACCELERATE: attributes #[[LOG10]] = { "vector-function-abi-variant"= +; ACCELERATE-SAME: "_ZGV_LLVM_N4v_llvm.log10.f32(vlog10f)" }