[TLI] Per-function fveclib for math library used for vectorization

Summary:
Encode `-fveclib` setting as per-function attribute so it can threaded through to LTO backends. Accordingly per-function TLI now reads
the attributes and select available vector function list based on that. Now we also populate function list for all supported vector
libraries for the shared per-module `TargetLibraryInfoImpl`, so each function can select its available vector list independently but without
duplicating the vector function lists. Inlining between incompatbile vectlib attributed is also prohibited now.

Subscribers: hiraditya, dexonsmith, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D77632

Author: WenleiHe

clang/lib/CodeGen/BackendUtil.cpp

@@ -352,24 +352,8 @@ static void addMemTagOptimizationPasses(const PassManagerBuilder &Builder,
   PM.add(createStackSafetyGlobalInfoWrapperPass(/*SetMetadata=*/true));
 }
 
-static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple,
-                                         const CodeGenOptions &CodeGenOpts) {
-  TargetLibraryInfoImpl *TLII = new TargetLibraryInfoImpl(TargetTriple);
-
-  switch (CodeGenOpts.getVecLib()) {
-  case CodeGenOptions::Accelerate:
-    TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate);
-    break;
-  case CodeGenOptions::MASSV:
-    TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::MASSV);
-    break;
-  case CodeGenOptions::SVML:
-    TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SVML);
-    break;
-  default:
-    break;
-  }
-  return TLII;
+static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple) {
+  return new TargetLibraryInfoImpl(TargetTriple);
 }
 
 static void addSymbolRewriterPass(const CodeGenOptions &Opts,
@@ -562,8 +546,7 @@ void EmitAssemblyHelper::CreatePasses(legacy::PassManager &MPM,
   // are inserted before PMBuilder ones - they'd get the default-constructed
   // TLI with an unknown target otherwise.
   Triple TargetTriple(TheModule->getTargetTriple());
-  std::unique_ptr<TargetLibraryInfoImpl> TLII(
-      createTLII(TargetTriple, CodeGenOpts));
+  std::unique_ptr<TargetLibraryInfoImpl> TLII(createTLII(TargetTriple));
 
   // If we reached here with a non-empty index file name, then the index file
   // was empty and we are not performing ThinLTO backend compilation (used in
@@ -805,8 +788,7 @@ bool EmitAssemblyHelper::AddEmitPasses(legacy::PassManager &CodeGenPasses,
                                        raw_pwrite_stream *DwoOS) {
   // Add LibraryInfo.
   llvm::Triple TargetTriple(TheModule->getTargetTriple());
-  std::unique_ptr<TargetLibraryInfoImpl> TLII(
-      createTLII(TargetTriple, CodeGenOpts));
+  std::unique_ptr<TargetLibraryInfoImpl> TLII(createTLII(TargetTriple));
   CodeGenPasses.add(new TargetLibraryInfoWrapperPass(*TLII));
 
   // Normal mode, emit a .s or .o file by running the code generator. Note,
@@ -1142,8 +1124,7 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
   // Register the target library analysis directly and give it a customized
   // preset TLI.
   Triple TargetTriple(TheModule->getTargetTriple());
-  std::unique_ptr<TargetLibraryInfoImpl> TLII(
-      createTLII(TargetTriple, CodeGenOpts));
+  std::unique_ptr<TargetLibraryInfoImpl> TLII(createTLII(TargetTriple));
   FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
 
   // Register all the basic analyses with the managers.

clang/lib/CodeGen/CGCall.cpp

@@ -1868,6 +1868,24 @@ static void addNoBuiltinAttributes(llvm::AttrBuilder &FuncAttrs,
   llvm::for_each(NBA->builtinNames(), AddNoBuiltinAttr);
 }
 
+static void addVectLibAttributes(llvm::AttrBuilder &FuncAttrs,
+                                 const CodeGenOptions &CodeGenOpts) {
+  StringRef AttributeName = "veclib";
+  switch (CodeGenOpts.getVecLib()) {
+  case CodeGenOptions::Accelerate:
+    FuncAttrs.addAttribute(AttributeName, "Accelerate");
+    break;
+  case CodeGenOptions::MASSV:
+    FuncAttrs.addAttribute(AttributeName, "MASSV");
+    break;
+  case CodeGenOptions::SVML:
+    FuncAttrs.addAttribute(AttributeName, "SVML");
+    break;
+  case CodeGenOptions::NoLibrary:
+    break;
+  }
+}
+
 void CodeGenModule::ConstructAttributeList(
     StringRef Name, const CGFunctionInfo &FI, CGCalleeInfo CalleeInfo,
     llvm::AttributeList &AttrList, unsigned &CallingConv, bool AttrOnCallSite) {
@@ -1966,6 +1984,9 @@ void CodeGenModule::ConstructAttributeList(
   // * FunctionDecl attributes: __attribute__((no_builtin(...)))
   addNoBuiltinAttributes(FuncAttrs, getLangOpts(), NBA);
 
+  // Attach "veclib" attribute to function based on '-fveclib' setting.
+  addVectLibAttributes(FuncAttrs, getCodeGenOpts());
+
   ConstructDefaultFnAttrList(Name, HasOptnone, AttrOnCallSite, FuncAttrs);
 
   // This must run after constructing the default function attribute list

clang/test/CodeGen/libcalls-veclib.c

@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -S -emit-llvm -fveclib=SVML -o - %s | FileCheck --check-prefixes=SVML %s
+// RUN: %clang_cc1 -S -emit-llvm -fveclib=Accelerate -o - %s | FileCheck --check-prefixes=ACCELERATE %s
+// RUN: %clang_cc1 -S -emit-llvm -fveclib=MASSV -o - %s | FileCheck --check-prefixes=MASSV %s
+// RUN: %clang_cc1 -S -emit-llvm -fveclib=none -o - %s | FileCheck --check-prefixes=NOLIB %s
+// RUN: %clang_cc1 -S -emit-llvm -o - %s | FileCheck --check-prefixes=NOLIB %s
+
+int main() {
+  return 0;
+}
+
+// SVML: "veclib"="SVML"
+// ACCELERATE: "veclib"="Accelerate"
+// MASSV: "veclib"="MASSV"
+// NOLIB-NOT: "veclib"

llvm/include/llvm/Analysis/TargetLibraryInfo.h

@@ -48,6 +48,23 @@ struct VecDesc {
 class TargetLibraryInfoImpl {
   friend class TargetLibraryInfo;
 
+public:
+  /// List of known vector-functions libraries.
+  ///
+  /// The vector-functions library defines, which functions are vectorizable
+  /// and with which factor. The library can be specified by either frontend,
+  /// or a commandline option, and then used by
+  /// addVectorizableFunctionsFromVecLib for filling up the tables of
+  /// vectorizable functions.
+  enum VectorLibrary {
+    Accelerate, // Use Accelerate framework.
+    MASSV,      // IBM MASS vector library.
+    SVML,       // Intel short vector math library.
+    NumVecLibs, // Number of supported vector libraries.
+    NoLibrary   // Don't use any vector library.
+  };
+
+private:
   unsigned char AvailableArray[(NumLibFuncs+3)/4];
   llvm::DenseMap<unsigned, std::string> CustomNames;
   static StringLiteral const StandardNames[NumLibFuncs];
@@ -66,32 +83,31 @@ class TargetLibraryInfoImpl {
     return static_cast<AvailabilityState>((AvailableArray[F/4] >> 2*(F&3)) & 3);
   }
 
-  /// Vectorization descriptors - sorted by ScalarFnName.
-  std::vector<VecDesc> VectorDescs;
-  /// Scalarization descriptors - same content as VectorDescs but sorted based
-  /// on VectorFnName rather than ScalarFnName.
-  std::vector<VecDesc> ScalarDescs;
+  /// Vector library descriptor for all supported ones.
+  struct VectorLibraryDescriptors {
+    /// Vectorization descriptors - sorted by ScalarFnName.
+    std::vector<VecDesc> VectorDescs;
+    /// Scalarization descriptors - same content as VectorDescs but sorted based
+    /// on VectorFnName rather than ScalarFnName.
+    std::vector<VecDesc> ScalarDescs;
+  } VecLibDescs[NumVecLibs];
 
   /// Return true if the function type FTy is valid for the library function
   /// F, regardless of whether the function is available.
   bool isValidProtoForLibFunc(const FunctionType &FTy, LibFunc F,
                               const DataLayout *DL) const;
 
-public:
-  /// List of known vector-functions libraries.
-  ///
-  /// The vector-functions library defines, which functions are vectorizable
-  /// and with which factor. The library can be specified by either frontend,
-  /// or a commandline option, and then used by
-  /// addVectorizableFunctionsFromVecLib for filling up the tables of
-  /// vectorizable functions.
-  enum VectorLibrary {
-    NoLibrary,  // Don't use any vector library.
-    Accelerate, // Use Accelerate framework.
-    MASSV,      // IBM MASS vector library.
-    SVML        // Intel short vector math library.
-  };
+  /// Add a set of scalar -> vector mappings, queryable via
+  /// getVectorizedFunction and getScalarizedFunction.
+  void addVectorizableFunctions(ArrayRef<VecDesc> Fns,
+                                VectorLibraryDescriptors &VetLibDescs);
 
+  /// Calls addVectorizableFunctionsFromVecLib with a known preset of functions
+  /// for the given vector library.
+  void addVectorizableFunctionsFromVecLib(enum VectorLibrary VecLib,
+                                          VectorLibraryDescriptors &VetLibDesc);
+
+public:
   TargetLibraryInfoImpl();
   explicit TargetLibraryInfoImpl(const Triple &T);
 
@@ -141,39 +157,38 @@ class TargetLibraryInfoImpl {
   /// This can be used for options like -fno-builtin.
   void disableAllFunctions();
 
-  /// Add a set of scalar -> vector mappings, queryable via
-  /// getVectorizedFunction and getScalarizedFunction.
-  void addVectorizableFunctions(ArrayRef<VecDesc> Fns);
-
-  /// Calls addVectorizableFunctions with a known preset of functions for the
-  /// given vector library.
-  void addVectorizableFunctionsFromVecLib(enum VectorLibrary VecLib);
+  /// Populate VectorLibraryDescriptors for all supported vector libraries.
+  void addAllVectorizableFunctions();
 
   /// Return true if the function F has a vector equivalent with vectorization
   /// factor VF.
-  bool isFunctionVectorizable(StringRef F, unsigned VF) const {
-    return !getVectorizedFunction(F, VF).empty();
+  bool isFunctionVectorizable(StringRef F, unsigned VF,
+                              VectorLibrary vecLib) const {
+    return !getVectorizedFunction(F, VF, vecLib).empty();
   }
 
   /// Return true if the function F has a vector equivalent with any
   /// vectorization factor.
-  bool isFunctionVectorizable(StringRef F) const;
+  bool isFunctionVectorizable(StringRef F, VectorLibrary vecLib) const;
 
   /// Return the name of the equivalent of F, vectorized with factor VF. If no
   /// such mapping exists, return the empty string.
-  StringRef getVectorizedFunction(StringRef F, unsigned VF) const;
+  StringRef getVectorizedFunction(StringRef F, unsigned VF,
+                                  VectorLibrary vecLib) const;
 
   /// Return true if the function F has a scalar equivalent, and set VF to be
   /// the vectorization factor.
-  bool isFunctionScalarizable(StringRef F, unsigned &VF) const {
-    return !getScalarizedFunction(F, VF).empty();
+  bool isFunctionScalarizable(StringRef F, unsigned &VF,
+                              VectorLibrary vecLib) const {
+    return !getScalarizedFunction(F, VF, vecLib).empty();
   }
 
   /// Return the name of the equivalent of F, scalarized. If no such mapping
   /// exists, return the empty string.
   ///
   /// Set VF to the vectorization factor.
-  StringRef getScalarizedFunction(StringRef F, unsigned &VF) const;
+  StringRef getScalarizedFunction(StringRef F, unsigned &VF,
+                                  VectorLibrary vecLib) const;
 
   /// Set to true iff i32 parameters to library functions should have signext
   /// or zeroext attributes if they correspond to C-level int or unsigned int,
@@ -201,7 +216,7 @@ class TargetLibraryInfoImpl {
 
   /// Returns the largest vectorization factor used in the list of
   /// vector functions.
-  unsigned getWidestVF(StringRef ScalarF) const;
+  unsigned getWidestVF(StringRef ScalarF, VectorLibrary vecLib) const;
 };
 
 /// Provides information about what library functions are available for
@@ -216,63 +231,66 @@ class TargetLibraryInfo {
   /// The global (module level) TLI info.
   const TargetLibraryInfoImpl *Impl;
 
+  /// Vector library available for vectorization.
+  TargetLibraryInfoImpl::VectorLibrary VectLibrary =
+      TargetLibraryInfoImpl::NoLibrary;
+
   /// Support for -fno-builtin* options as function attributes, overrides
   /// information in global TargetLibraryInfoImpl.
   BitVector OverrideAsUnavailable;
 
+  TargetLibraryInfoImpl::VectorLibrary
+  getVecLibFromName(const StringRef &VecLibName) {
+    if (VecLibName == "Accelerate")
+      return TargetLibraryInfoImpl::Accelerate;
+    else if (VecLibName == "MASSV")
+      return TargetLibraryInfoImpl::MASSV;
+    else if (VecLibName == "SVML")
+      return TargetLibraryInfoImpl::SVML;
+    return TargetLibraryInfoImpl::NoLibrary;
+  }
+
 public:
   explicit TargetLibraryInfo(const TargetLibraryInfoImpl &Impl,
-                             Optional<const Function *> F = None)
-      : Impl(&Impl), OverrideAsUnavailable(NumLibFuncs) {
-    if (!F)
-      return;
-    if ((*F)->hasFnAttribute("no-builtins"))
-      disableAllFunctions();
-    else {
-      // Disable individual libc/libm calls in TargetLibraryInfo.
-      LibFunc LF;
-      AttributeSet FnAttrs = (*F)->getAttributes().getFnAttributes();
-      for (const Attribute &Attr : FnAttrs) {
-        if (!Attr.isStringAttribute())
-          continue;
-        auto AttrStr = Attr.getKindAsString();
-        if (!AttrStr.consume_front("no-builtin-"))
-          continue;
-        if (getLibFunc(AttrStr, LF))
-          setUnavailable(LF);
-      }
-    }
-  }
+                             Optional<const Function *> F = None);
 
   // Provide value semantics.
   TargetLibraryInfo(const TargetLibraryInfo &TLI)
-      : Impl(TLI.Impl), OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
+      : Impl(TLI.Impl), VectLibrary(TLI.VectLibrary),
+        OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
   TargetLibraryInfo(TargetLibraryInfo &&TLI)
-      : Impl(TLI.Impl), OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
+      : Impl(TLI.Impl), VectLibrary(TLI.VectLibrary),
+        OverrideAsUnavailable(TLI.OverrideAsUnavailable) {}
   TargetLibraryInfo &operator=(const TargetLibraryInfo &TLI) {
     Impl = TLI.Impl;
+    VectLibrary = TLI.VectLibrary;
     OverrideAsUnavailable = TLI.OverrideAsUnavailable;
     return *this;
   }
   TargetLibraryInfo &operator=(TargetLibraryInfo &&TLI) {
     Impl = TLI.Impl;
+    VectLibrary = TLI.VectLibrary;
     OverrideAsUnavailable = TLI.OverrideAsUnavailable;
     return *this;
   }
 
   /// Determine whether a callee with the given TLI can be inlined into
-  /// caller with this TLI, based on 'nobuiltin' attributes. When requested,
-  /// allow inlining into a caller with a superset of the callee's nobuiltin
-  /// attributes, which is conservatively correct.
+  /// caller with this TLI, based on 'nobuiltin', `veclib` attributes.
+  /// When requested, allow inlining into a caller with a superset of the
+  /// callee's attributes, which is conservatively correct.
   bool areInlineCompatible(const TargetLibraryInfo &CalleeTLI,
                            bool AllowCallerSuperset) const {
     if (!AllowCallerSuperset)
-      return OverrideAsUnavailable == CalleeTLI.OverrideAsUnavailable;
+      return VectLibrary == CalleeTLI.VectLibrary &&
+             OverrideAsUnavailable == CalleeTLI.OverrideAsUnavailable;
     BitVector B = OverrideAsUnavailable;
     B |= CalleeTLI.OverrideAsUnavailable;
-    // We can inline if the union of the caller and callee's nobuiltin
-    // attributes is no stricter than the caller's nobuiltin attributes.
-    return B == OverrideAsUnavailable;
+    // We can inline if the union of the caller and callee's attributes
+    // is no stricter than the caller's attributes.
+    bool VecLibCompatible =
+        (VectLibrary == CalleeTLI.VectLibrary) ||
+        CalleeTLI.VectLibrary == TargetLibraryInfoImpl::NoLibrary;
+    return B == OverrideAsUnavailable && VecLibCompatible;
   }
 
   /// Searches for a particular function name.
@@ -317,13 +335,13 @@ class TargetLibraryInfo {
     return getState(F) != TargetLibraryInfoImpl::Unavailable;
   }
   bool isFunctionVectorizable(StringRef F, unsigned VF) const {
-    return Impl->isFunctionVectorizable(F, VF);
+    return Impl->isFunctionVectorizable(F, VF, VectLibrary);
   }
   bool isFunctionVectorizable(StringRef F) const {
-    return Impl->isFunctionVectorizable(F);
+    return Impl->isFunctionVectorizable(F, VectLibrary);
   }
   StringRef getVectorizedFunction(StringRef F, unsigned VF) const {
-    return Impl->getVectorizedFunction(F, VF);
+    return Impl->getVectorizedFunction(F, VF, VectLibrary);
   }
 
   /// Tests if the function is both available and a candidate for optimized code
@@ -408,7 +426,7 @@ class TargetLibraryInfo {
   /// Returns the largest vectorization factor used in the list of
   /// vector functions.
   unsigned getWidestVF(StringRef ScalarF) const {
-    return Impl->getWidestVF(ScalarF);
+    return Impl->getWidestVF(ScalarF, VectLibrary);
   }
 
   /// Check if the function "F" is listed in a library known to LLVM.

llvm/lib/Analysis/InlineCost.cpp

@@ -104,10 +104,9 @@ static cl::opt<bool> OptComputeFullInlineCost(
     cl::desc("Compute the full inline cost of a call site even when the cost "
              "exceeds the threshold."));
 
-static cl::opt<bool> InlineCallerSupersetNoBuiltin(
-    "inline-caller-superset-nobuiltin", cl::Hidden, cl::init(true),
-    cl::ZeroOrMore,
-    cl::desc("Allow inlining when caller has a superset of callee's nobuiltin "
+static cl::opt<bool> InlineCallerSupersetTLI(
+    "inline-caller-superset-tli", cl::Hidden, cl::init(true), cl::ZeroOrMore,
+    cl::desc("Allow inlining when caller has a superset of callee's TLI "
              "attributes."));
 
 namespace {
@@ -2169,7 +2168,7 @@ static bool functionsHaveCompatibleAttributes(
   auto CalleeTLI = GetTLI(*Callee);
   return TTI.areInlineCompatible(Caller, Callee) &&
          GetTLI(*Caller).areInlineCompatible(CalleeTLI,
-                                             InlineCallerSupersetNoBuiltin) &&
+                                             InlineCallerSupersetTLI) &&
          AttributeFuncs::areInlineCompatible(*Caller, *Callee);
 }