Skip to content

Commit

Permalink
[TLI] Per-function fveclib for math library used for vectorization
Browse files Browse the repository at this point in the history
Summary:
Encode `-fveclib` setting as per-function attribute so it can threaded through to LTO backends. Accordingly per-function TLI now reads
the attributes and select available vector function list based on that. Now we also populate function list for all supported vector
libraries for the shared per-module `TargetLibraryInfoImpl`, so each function can select its available vector list independently but without
duplicating the vector function lists. Inlining between incompatbile vectlib attributed is also prohibited now.

Subscribers: hiraditya, dexonsmith, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D77632
  • Loading branch information
WenleiHe committed Apr 10, 2020
1 parent 5b18b6e commit 60c642e
Show file tree
Hide file tree
Showing 8 changed files with 250 additions and 120 deletions.
29 changes: 5 additions & 24 deletions clang/lib/CodeGen/BackendUtil.cpp
Expand Up @@ -352,24 +352,8 @@ static void addMemTagOptimizationPasses(const PassManagerBuilder &Builder,
PM.add(createStackSafetyGlobalInfoWrapperPass(/*SetMetadata=*/true));
}

static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple,
const CodeGenOptions &CodeGenOpts) {
TargetLibraryInfoImpl *TLII = new TargetLibraryInfoImpl(TargetTriple);

switch (CodeGenOpts.getVecLib()) {
case CodeGenOptions::Accelerate:
TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate);
break;
case CodeGenOptions::MASSV:
TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::MASSV);
break;
case CodeGenOptions::SVML:
TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SVML);
break;
default:
break;
}
return TLII;
static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple) {
return new TargetLibraryInfoImpl(TargetTriple);
}

static void addSymbolRewriterPass(const CodeGenOptions &Opts,
Expand Down Expand Up @@ -562,8 +546,7 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM,
// are inserted before PMBuilder ones - they'd get the default-constructed
// TLI with an unknown target otherwise.
Triple TargetTriple(TheModule->getTargetTriple());
std::unique_ptr<TargetLibraryInfoImpl> TLII(
createTLII(TargetTriple, CodeGenOpts));
std::unique_ptr<TargetLibraryInfoImpl> TLII(createTLII(TargetTriple));

// If we reached here with a non-empty index file name, then the index file
// was empty and we are not performing ThinLTO backend compilation (used in
Expand Down Expand Up @@ -805,8 +788,7 @@ bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses,
raw_pwrite_stream *DwoOS) {
// Add LibraryInfo.
llvm::Triple TargetTriple(TheModule->getTargetTriple());
std::unique_ptr<TargetLibraryInfoImpl> TLII(
createTLII(TargetTriple, CodeGenOpts));
std::unique_ptr<TargetLibraryInfoImpl> TLII(createTLII(TargetTriple));
CodeGenPasses.add(new TargetLibraryInfoWrapperPass(*TLII));

// Normal mode, emit a .s or .o file by running the code generator. Note,
Expand Down Expand Up @@ -1142,8 +1124,7 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
// Register the target library analysis directly and give it a customized
// preset TLI.
Triple TargetTriple(TheModule->getTargetTriple());
std::unique_ptr<TargetLibraryInfoImpl> TLII(
createTLII(TargetTriple, CodeGenOpts));
std::unique_ptr<TargetLibraryInfoImpl> TLII(createTLII(TargetTriple));
FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });

// Register all the basic analyses with the managers.
Expand Down
21 changes: 21 additions & 0 deletions clang/lib/CodeGen/CGCall.cpp
Expand Up @@ -1868,6 +1868,24 @@ static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
}

static void addVectLibAttributes(llvm::AttrBuilder &FuncAttrs,
const CodeGenOptions &CodeGenOpts) {
StringRef AttributeName = "veclib";
switch (CodeGenOpts.getVecLib()) {
case CodeGenOptions::Accelerate:
FuncAttrs.addAttribute(AttributeName, "Accelerate");
break;
case CodeGenOptions::MASSV:
FuncAttrs.addAttribute(AttributeName, "MASSV");
break;
case CodeGenOptions::SVML:
FuncAttrs.addAttribute(AttributeName, "SVML");
break;
case CodeGenOptions::NoLibrary:
break;
}
}

void CodeGenModule::ConstructAttributeList(
StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
Expand Down Expand Up @@ -1966,6 +1984,9 @@ void CodeGenModule::ConstructAttributeList(
// * FunctionDecl attributes: __attribute__((no_builtin(...)))
addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);

// Attach "veclib" attribute to function based on '-fveclib' setting.
addVectLibAttributes(FuncAttrs, getCodeGenOpts());

ConstructDefaultFnAttrList(Name, HasOptnone, AttrOnCallSite, FuncAttrs);

// This must run after constructing the default function attribute list
Expand Down
14 changes: 14 additions & 0 deletions clang/test/CodeGen/libcalls-veclib.c
@@ -0,0 +1,14 @@
// RUN: %clang_cc1 -S -emit-llvm -fveclib=SVML -o - %s | FileCheck --check-prefixes=SVML %s
// RUN: %clang_cc1 -S -emit-llvm -fveclib=Accelerate -o - %s | FileCheck --check-prefixes=ACCELERATE %s
// RUN: %clang_cc1 -S -emit-llvm -fveclib=MASSV -o - %s | FileCheck --check-prefixes=MASSV %s
// RUN: %clang_cc1 -S -emit-llvm -fveclib=none -o - %s | FileCheck --check-prefixes=NOLIB %s
// RUN: %clang_cc1 -S -emit-llvm -o - %s | FileCheck --check-prefixes=NOLIB %s

int main() {
return 0;
}

// SVML: "veclib"="SVML"
// ACCELERATE: "veclib"="Accelerate"
// MASSV: "veclib"="MASSV"
// NOLIB-NOT: "veclib"
154 changes: 86 additions & 68 deletions llvm/include/llvm/Analysis/TargetLibraryInfo.h
Expand Up @@ -48,6 +48,23 @@ struct VecDesc {
class TargetLibraryInfoImpl {
friend class TargetLibraryInfo;

public:
/// List of known vector-functions libraries.
///
/// The vector-functions library defines, which functions are vectorizable
/// and with which factor. The library can be specified by either frontend,
/// or a commandline option, and then used by
/// addVectorizableFunctionsFromVecLib for filling up the tables of
/// vectorizable functions.
enum VectorLibrary {
Accelerate, // Use Accelerate framework.
MASSV, // IBM MASS vector library.
SVML, // Intel short vector math library.
NumVecLibs, // Number of supported vector libraries.
NoLibrary // Don't use any vector library.
};

private:
unsigned char AvailableArray[(NumLibFuncs+3)/4];
llvm::DenseMap<unsigned, std::string> CustomNames;
static StringLiteral const StandardNames[NumLibFuncs];
Expand All @@ -66,32 +83,31 @@ class TargetLibraryInfoImpl {
return static_cast<AvailabilityState>((AvailableArray[F/4] >> 2*(F&3)) & 3);
}

/// Vectorization descriptors - sorted by ScalarFnName.
std::vector<VecDesc> VectorDescs;
/// Scalarization descriptors - same content as VectorDescs but sorted based
/// on VectorFnName rather than ScalarFnName.
std::vector<VecDesc> ScalarDescs;
/// Vector library descriptor for all supported ones.
struct VectorLibraryDescriptors {
/// Vectorization descriptors - sorted by ScalarFnName.
std::vector<VecDesc> VectorDescs;
/// Scalarization descriptors - same content as VectorDescs but sorted based
/// on VectorFnName rather than ScalarFnName.
std::vector<VecDesc> ScalarDescs;
} VecLibDescs[NumVecLibs];

/// Return true if the function type FTy is valid for the library function
/// F, regardless of whether the function is available.
bool isValidProtoForLibFunc(const FunctionType &FTy, LibFunc F,
const DataLayout *DL) const;

public:
/// List of known vector-functions libraries.
///
/// The vector-functions library defines, which functions are vectorizable
/// and with which factor. The library can be specified by either frontend,
/// or a commandline option, and then used by
/// addVectorizableFunctionsFromVecLib for filling up the tables of
/// vectorizable functions.
enum VectorLibrary {
NoLibrary, // Don't use any vector library.
Accelerate, // Use Accelerate framework.
MASSV, // IBM MASS vector library.
SVML // Intel short vector math library.
};
/// Add a set of scalar -> vector mappings, queryable via
/// getVectorizedFunction and getScalarizedFunction.
void addVectorizableFunctions(ArrayRef<VecDesc> Fns,
VectorLibraryDescriptors &VetLibDescs);

/// Calls addVectorizableFunctionsFromVecLib with a known preset of functions
/// for the given vector library.
void addVectorizableFunctionsFromVecLib(enum VectorLibrary VecLib,
VectorLibraryDescriptors &VetLibDesc);

public:
TargetLibraryInfoImpl();
explicit TargetLibraryInfoImpl(const Triple &T);

Expand Down Expand Up @@ -141,39 +157,38 @@ class TargetLibraryInfoImpl {
/// This can be used for options like -fno-builtin.
void disableAllFunctions();

/// Add a set of scalar -> vector mappings, queryable via
/// getVectorizedFunction and getScalarizedFunction.
void addVectorizableFunctions(ArrayRef<VecDesc> Fns);

/// Calls addVectorizableFunctions with a known preset of functions for the
/// given vector library.
void addVectorizableFunctionsFromVecLib(enum VectorLibrary VecLib);
/// Populate VectorLibraryDescriptors for all supported vector libraries.
void addAllVectorizableFunctions();

/// Return true if the function F has a vector equivalent with vectorization
/// factor VF.
bool isFunctionVectorizable(StringRef F, unsigned VF) const {
return !getVectorizedFunction(F, VF).empty();
bool isFunctionVectorizable(StringRef F, unsigned VF,
VectorLibrary vecLib) const {
return !getVectorizedFunction(F, VF, vecLib).empty();
}

/// Return true if the function F has a vector equivalent with any
/// vectorization factor.
bool isFunctionVectorizable(StringRef F) const;
bool isFunctionVectorizable(StringRef F, VectorLibrary vecLib) const;

/// Return the name of the equivalent of F, vectorized with factor VF. If no
/// such mapping exists, return the empty string.
StringRef getVectorizedFunction(StringRef F, unsigned VF) const;
StringRef getVectorizedFunction(StringRef F, unsigned VF,
VectorLibrary vecLib) const;

/// Return true if the function F has a scalar equivalent, and set VF to be
/// the vectorization factor.
bool isFunctionScalarizable(StringRef F, unsigned &VF) const {
return !getScalarizedFunction(F, VF).empty();
bool isFunctionScalarizable(StringRef F, unsigned &VF,
VectorLibrary vecLib) const {
return !getScalarizedFunction(F, VF, vecLib).empty();
}

/// Return the name of the equivalent of F, scalarized. If no such mapping
/// exists, return the empty string.
///
/// Set VF to the vectorization factor.
StringRef getScalarizedFunction(StringRef F, unsigned &VF) const;
StringRef getScalarizedFunction(StringRef F, unsigned &VF,
VectorLibrary vecLib) const;

/// Set to true iff i32 parameters to library functions should have signext
/// or zeroext attributes if they correspond to C-level int or unsigned int,
Expand Down Expand Up @@ -201,7 +216,7 @@ class TargetLibraryInfoImpl {

/// Returns the largest vectorization factor used in the list of
/// vector functions.
unsigned getWidestVF(StringRef ScalarF) const;
unsigned getWidestVF(StringRef ScalarF, VectorLibrary vecLib) const;
};

/// Provides information about what library functions are available for
Expand All @@ -216,63 +231,66 @@ class TargetLibraryInfo {
/// The global (module level) TLI info.
const TargetLibraryInfoImpl *Impl;

/// Vector library available for vectorization.
TargetLibraryInfoImpl::VectorLibrary VectLibrary =
TargetLibraryInfoImpl::NoLibrary;

/// Support for -fno-builtin* options as function attributes, overrides
/// information in global TargetLibraryInfoImpl.
BitVector OverrideAsUnavailable;

TargetLibraryInfoImpl::VectorLibrary
getVecLibFromName(const StringRef &VecLibName) {
if (VecLibName == "Accelerate")
return TargetLibraryInfoImpl::Accelerate;
else if (VecLibName == "MASSV")
return TargetLibraryInfoImpl::MASSV;
else if (VecLibName == "SVML")
return TargetLibraryInfoImpl::SVML;
return TargetLibraryInfoImpl::NoLibrary;
}

public:
explicit TargetLibraryInfo(const TargetLibraryInfoImpl &Impl,
Optional<const Function *> F = None)
: Impl(&Impl), OverrideAsUnavailable(NumLibFuncs) {
if (!F)
return;
if ((*F)->hasFnAttribute("no-builtins"))
disableAllFunctions();
else {
// Disable individual libc/libm calls in TargetLibraryInfo.
LibFunc LF;
AttributeSet FnAttrs = (*F)->getAttributes().getFnAttributes();
for (const Attribute &Attr : FnAttrs) {
if (!Attr.isStringAttribute())
continue;
auto AttrStr = Attr.getKindAsString();
if (!AttrStr.consume_front("no-builtin-"))
continue;
if (getLibFunc(AttrStr, LF))
setUnavailable(LF);
}
}
}
Optional<const Function *> F = None);

// Provide value semantics.
TargetLibraryInfo(const TargetLibraryInfo &TLI)
: Impl(TLI.Impl), OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
: Impl(TLI.Impl), VectLibrary(TLI.VectLibrary),
OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
TargetLibraryInfo(TargetLibraryInfo &&TLI)
: Impl(TLI.Impl), OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
: Impl(TLI.Impl), VectLibrary(TLI.VectLibrary),
OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
TargetLibraryInfo &operator=(const TargetLibraryInfo &TLI) {
Impl = TLI.Impl;
VectLibrary = TLI.VectLibrary;
OverrideAsUnavailable = TLI.OverrideAsUnavailable;
return *this;
}
TargetLibraryInfo &operator=(TargetLibraryInfo &&TLI) {
Impl = TLI.Impl;
VectLibrary = TLI.VectLibrary;
OverrideAsUnavailable = TLI.OverrideAsUnavailable;
return *this;
}

/// Determine whether a callee with the given TLI can be inlined into
/// caller with this TLI, based on 'nobuiltin' attributes. When requested,
/// allow inlining into a caller with a superset of the callee's nobuiltin
/// attributes, which is conservatively correct.
/// caller with this TLI, based on 'nobuiltin', `veclib` attributes.
/// When requested, allow inlining into a caller with a superset of the
/// callee's attributes, which is conservatively correct.
bool areInlineCompatible(const TargetLibraryInfo &CalleeTLI,
bool AllowCallerSuperset) const {
if (!AllowCallerSuperset)
return OverrideAsUnavailable == CalleeTLI.OverrideAsUnavailable;
return VectLibrary == CalleeTLI.VectLibrary &&
OverrideAsUnavailable == CalleeTLI.OverrideAsUnavailable;
BitVector B = OverrideAsUnavailable;
B |= CalleeTLI.OverrideAsUnavailable;
// We can inline if the union of the caller and callee's nobuiltin
// attributes is no stricter than the caller's nobuiltin attributes.
return B == OverrideAsUnavailable;
// We can inline if the union of the caller and callee's attributes
// is no stricter than the caller's attributes.
bool VecLibCompatible =
(VectLibrary == CalleeTLI.VectLibrary) ||
CalleeTLI.VectLibrary == TargetLibraryInfoImpl::NoLibrary;
return B == OverrideAsUnavailable && VecLibCompatible;
}

/// Searches for a particular function name.
Expand Down Expand Up @@ -317,13 +335,13 @@ class TargetLibraryInfo {
return getState(F) != TargetLibraryInfoImpl::Unavailable;
}
bool isFunctionVectorizable(StringRef F, unsigned VF) const {
return Impl->isFunctionVectorizable(F, VF);
return Impl->isFunctionVectorizable(F, VF, VectLibrary);
}
bool isFunctionVectorizable(StringRef F) const {
return Impl->isFunctionVectorizable(F);
return Impl->isFunctionVectorizable(F, VectLibrary);
}
StringRef getVectorizedFunction(StringRef F, unsigned VF) const {
return Impl->getVectorizedFunction(F, VF);
return Impl->getVectorizedFunction(F, VF, VectLibrary);
}

/// Tests if the function is both available and a candidate for optimized code
Expand Down Expand Up @@ -408,7 +426,7 @@ class TargetLibraryInfo {
/// Returns the largest vectorization factor used in the list of
/// vector functions.
unsigned getWidestVF(StringRef ScalarF) const {
return Impl->getWidestVF(ScalarF);
return Impl->getWidestVF(ScalarF, VectLibrary);
}

/// Check if the function "F" is listed in a library known to LLVM.
Expand Down
9 changes: 4 additions & 5 deletions llvm/lib/Analysis/InlineCost.cpp
Expand Up @@ -104,10 +104,9 @@ static cl::opt<bool> OptComputeFullInlineCost(
cl::desc("Compute the full inline cost of a call site even when the cost "
"exceeds the threshold."));

static cl::opt<bool> InlineCallerSupersetNoBuiltin(
"inline-caller-superset-nobuiltin", cl::Hidden, cl::init(true),
cl::ZeroOrMore,
cl::desc("Allow inlining when caller has a superset of callee's nobuiltin "
static cl::opt<bool> InlineCallerSupersetTLI(
"inline-caller-superset-tli", cl::Hidden, cl::init(true), cl::ZeroOrMore,
cl::desc("Allow inlining when caller has a superset of callee's TLI "
"attributes."));

namespace {
Expand Down Expand Up @@ -2169,7 +2168,7 @@ static bool functionsHaveCompatibleAttributes(
auto CalleeTLI = GetTLI(*Callee);
return TTI.areInlineCompatible(Caller, Callee) &&
GetTLI(*Caller).areInlineCompatible(CalleeTLI,
InlineCallerSupersetNoBuiltin) &&
InlineCallerSupersetTLI) &&
AttributeFuncs::areInlineCompatible(*Caller, *Callee);
}

Expand Down

0 comments on commit 60c642e

Please sign in to comment.