WholeProgramDevirt: introduce.

This pass implements whole program optimization of virtual calls in cases
where we know (via bitset information) that the list of callees is fixed. This
includes the following:

- Single implementation devirtualization: if a virtual call has a single
  possible callee, replace all calls with a direct call to that callee.

- Virtual constant propagation: if the virtual function's return type is an
  integer <=64 bits and all possible callees are readnone, for each class and
  each list of constant arguments: evaluate the function, store the return
  value alongside the virtual table, and rewrite each virtual call as a load
  from the virtual table.

- Uniform return value optimization: if the conditions for virtual constant
  propagation hold and each function returns the same constant value, replace
  each virtual call with that constant.

- Unique return value optimization for i1 return values: if the conditions
  for virtual constant propagation hold and a single vtable's function
  returns 0, or a single vtable's function returns 1, replace each virtual
  call with a comparison of the vptr against that vtable's address.

Differential Revision: http://reviews.llvm.org/D16795

llvm-svn: 260312

llvm/include/llvm/InitializePasses.h

@@ -316,6 +316,7 @@ void initializeFuncletLayoutPass(PassRegistry &);
 void initializeLoopLoadEliminationPass(PassRegistry&);
 void initializeFunctionImportPassPass(PassRegistry &);
 void initializeLoopVersioningPassPass(PassRegistry &);
+void initializeWholeProgramDevirtPass(PassRegistry &);
 }
 
 #endif

llvm/include/llvm/Transforms/IPO.h

@@ -226,6 +226,10 @@ ModulePass *createLowerBitSetsPass();
 /// \brief This pass export CFI checks for use by external modules.
 ModulePass *createCrossDSOCFIPass();
 
+/// \brief This pass implements whole-program devirtualization using bitset
+/// metadata.
+ModulePass *createWholeProgramDevirtPass();
+
 //===----------------------------------------------------------------------===//
 // SampleProfilePass - Loads sample profile data from disk and generates
 // IR metadata to reflect the profile.

llvm/include/llvm/Transforms/IPO/PassManagerBuilder.h

@@ -157,6 +157,7 @@ class PassManagerBuilder {
                          legacy::PassManagerBase &PM) const;
   void addInitialAliasAnalysisPasses(legacy::PassManagerBase &PM) const;
   void addLTOOptimizationPasses(legacy::PassManagerBase &PM);
+  void addEarlyLTOOptimizationPasses(legacy::PassManagerBase &PM);
   void addLateLTOOptimizationPasses(legacy::PassManagerBase &PM);
   void addPGOInstrPasses(legacy::PassManagerBase &MPM);
 

llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h

@@ -0,0 +1,215 @@
+//===- WholeProgramDevirt.h - Whole-program devirt pass ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines parts of the whole-program devirtualization pass
+// implementation that may be usefully unit tested.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_WHOLEPROGRAMDEVIRT_H
+#define LLVM_TRANSFORMS_IPO_WHOLEPROGRAMDEVIRT_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMapInfo.h"
+#include <utility>
+#include <vector>
+#include <assert.h>
+#include <stdint.h>
+
+namespace llvm {
+
+class Function;
+class GlobalVariable;
+
+namespace wholeprogramdevirt {
+
+// A bit vector that keeps track of which bits are used. We use this to
+// pack constant values compactly before and after each virtual table.
+struct AccumBitVector {
+  std::vector<uint8_t> Bytes;
+
+  // Bits in BytesUsed[I] are 1 if matching bit in Bytes[I] is used, 0 if not.
+  std::vector<uint8_t> BytesUsed;
+
+  std::pair<uint8_t *, uint8_t *> getPtrToData(uint64_t Pos, uint8_t Size) {
+    if (Bytes.size() < Pos + Size) {
+      Bytes.resize(Pos + Size);
+      BytesUsed.resize(Pos + Size);
+    }
+    return std::make_pair(Bytes.data() + Pos, BytesUsed.data() + Pos);
+  }
+
+  // Set little-endian value Val with size Size at bit position Pos,
+  // and mark bytes as used.
+  void setLE(uint64_t Pos, uint64_t Val, uint8_t Size) {
+    assert(Pos % 8 == 0);
+    auto DataUsed = getPtrToData(Pos / 8, Size);
+    for (unsigned I = 0; I != Size; ++I) {
+      DataUsed.first[I] = Val >> (I * 8);
+      assert(!DataUsed.second[I]);
+      DataUsed.second[I] = 0xff;
+    }
+  }
+
+  // Set big-endian value Val with size Size at bit position Pos,
+  // and mark bytes as used.
+  void setBE(uint64_t Pos, uint64_t Val, uint8_t Size) {
+    assert(Pos % 8 == 0);
+    auto DataUsed = getPtrToData(Pos / 8, Size);
+    for (unsigned I = 0; I != Size; ++I) {
+      DataUsed.first[Size - I - 1] = Val >> (I * 8);
+      assert(!DataUsed.second[Size - I - 1]);
+      DataUsed.second[Size - I - 1] = 0xff;
+    }
+  }
+
+  // Set bit at bit position Pos to b and mark bit as used.
+  void setBit(uint64_t Pos, bool b) {
+    auto DataUsed = getPtrToData(Pos / 8, 1);
+    if (b)
+      *DataUsed.first |= 1 << (Pos % 8);
+    assert(!(*DataUsed.second & (1 << Pos % 8)));
+    *DataUsed.second |= 1 << (Pos % 8);
+  }
+};
+
+// The bits that will be stored before and after a particular vtable.
+struct VTableBits {
+  // The vtable global.
+  GlobalVariable *GV;
+
+  // Cache of the vtable's size in bytes.
+  uint64_t ObjectSize = 0;
+
+  // The bit vector that will be laid out before the vtable. Note that these
+  // bytes are stored in reverse order until the globals are rebuilt. This means
+  // that any values in the array must be stored using the opposite endianness
+  // from the target.
+  AccumBitVector Before;
+
+  // The bit vector that will be laid out after the vtable.
+  AccumBitVector After;
+};
+
+// Information about an entry in a particular bitset.
+struct BitSetInfo {
+  // The VTableBits for the vtable.
+  VTableBits *Bits;
+
+  // The offset in bytes from the start of the vtable (i.e. the address point).
+  uint64_t Offset;
+
+  bool operator<(const BitSetInfo &other) const {
+    return Bits < other.Bits || (Bits == other.Bits && Offset < other.Offset);
+  }
+};
+
+// A virtual call target, i.e. an entry in a particular vtable.
+struct VirtualCallTarget {
+  VirtualCallTarget(Function *Fn, const BitSetInfo *BS);
+
+  // For testing only.
+  VirtualCallTarget(const BitSetInfo *BS, bool IsBigEndian)
+      : Fn(nullptr), BS(BS), IsBigEndian(IsBigEndian) {}
+
+  // The function stored in the vtable.
+  Function *Fn;
+
+  // A pointer to the bitset through which the pointer to Fn is accessed.
+  const BitSetInfo *BS;
+
+  // When doing virtual constant propagation, this stores the return value for
+  // the function when passed the currently considered argument list.
+  uint64_t RetVal;
+
+  // Whether the target is big endian.
+  bool IsBigEndian;
+
+  // The minimum byte offset before the address point. This covers the bytes in
+  // the vtable object before the address point (e.g. RTTI, access-to-top,
+  // vtables for other base classes) and is equal to the offset from the start
+  // of the vtable object to the address point.
+  uint64_t minBeforeBytes() const { return BS->Offset; }
+
+  // The minimum byte offset after the address point. This covers the bytes in
+  // the vtable object after the address point (e.g. the vtable for the current
+  // class and any later base classes) and is equal to the size of the vtable
+  // object minus the offset from the start of the vtable object to the address
+  // point.
+  uint64_t minAfterBytes() const { return BS->Bits->ObjectSize - BS->Offset; }
+
+  // The number of bytes allocated (for the vtable plus the byte array) before
+  // the address point.
+  uint64_t allocatedBeforeBytes() const {
+    return minBeforeBytes() + BS->Bits->Before.Bytes.size();
+  }
+
+  // The number of bytes allocated (for the vtable plus the byte array) after
+  // the address point.
+  uint64_t allocatedAfterBytes() const {
+    return minAfterBytes() + BS->Bits->After.Bytes.size();
+  }
+
+  // Set the bit at position Pos before the address point to RetVal.
+  void setBeforeBit(uint64_t Pos) {
+    assert(Pos >= 8 * minBeforeBytes());
+    BS->Bits->Before.setBit(Pos - 8 * minBeforeBytes(), RetVal);
+  }
+
+  // Set the bit at position Pos after the address point to RetVal.
+  void setAfterBit(uint64_t Pos) {
+    assert(Pos >= 8 * minAfterBytes());
+    BS->Bits->After.setBit(Pos - 8 * minAfterBytes(), RetVal);
+  }
+
+  // Set the bytes at position Pos before the address point to RetVal.
+  // Because the bytes in Before are stored in reverse order, we use the
+  // opposite endianness to the target.
+  void setBeforeBytes(uint64_t Pos, uint8_t Size) {
+    assert(Pos >= 8 * minBeforeBytes());
+    if (IsBigEndian)
+      BS->Bits->Before.setLE(Pos - 8 * minBeforeBytes(), RetVal, Size);
+    else
+      BS->Bits->Before.setBE(Pos - 8 * minBeforeBytes(), RetVal, Size);
+  }
+
+  // Set the bytes at position Pos after the address point to RetVal.
+  void setAfterBytes(uint64_t Pos, uint8_t Size) {
+    assert(Pos >= 8 * minAfterBytes());
+    if (IsBigEndian)
+      BS->Bits->After.setBE(Pos - 8 * minAfterBytes(), RetVal, Size);
+    else
+      BS->Bits->After.setLE(Pos - 8 * minAfterBytes(), RetVal, Size);
+  }
+};
+
+// Find the minimum offset that we may store a value of size Size bits at. If
+// IsAfter is set, look for an offset before the object, otherwise look for an
+// offset after the object.
+uint64_t findLowestOffset(ArrayRef<VirtualCallTarget> Targets, bool IsAfter,
+                          uint64_t Size);
+
+// Set the stored value in each of Targets to VirtualCallTarget::RetVal at the
+// given allocation offset before the vtable address. Stores the computed
+// byte/bit offset to OffsetByte/OffsetBit.
+void setBeforeReturnValues(MutableArrayRef<VirtualCallTarget> Targets,
+                           uint64_t AllocBefore, unsigned BitWidth,
+                           int64_t &OffsetByte, uint64_t &OffsetBit);
+
+// Set the stored value in each of Targets to VirtualCallTarget::RetVal at the
+// given allocation offset after the vtable address. Stores the computed
+// byte/bit offset to OffsetByte/OffsetBit.
+void setAfterReturnValues(MutableArrayRef<VirtualCallTarget> Targets,
+                          uint64_t AllocAfter, unsigned BitWidth,
+                          int64_t &OffsetByte, uint64_t &OffsetBit);
+
+}
+}
+
+#endif

llvm/lib/Transforms/IPO/CMakeLists.txt

@@ -27,6 +27,7 @@ add_llvm_library(LLVMipo
   SampleProfile.cpp
   StripDeadPrototypes.cpp
   StripSymbols.cpp
+  WholeProgramDevirt.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms

llvm/lib/Transforms/IPO/IPO.cpp

@@ -53,6 +53,7 @@ void llvm::initializeIPO(PassRegistry &Registry) {
   initializeEliminateAvailableExternallyPass(Registry);
   initializeSampleProfileLoaderPass(Registry);
   initializeFunctionImportPassPass(Registry);
+  initializeWholeProgramDevirtPass(Registry);
 }
 
 void LLVMInitializeIPO(LLVMPassRegistryRef R) {

llvm/lib/Transforms/IPO/PassManagerBuilder.cpp

@@ -651,6 +651,16 @@ void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
   PM.add(createJumpThreadingPass());
 }
 
+void PassManagerBuilder::addEarlyLTOOptimizationPasses(
+    legacy::PassManagerBase &PM) {
+  // Remove unused virtual tables to improve the quality of code generated by
+  // whole-program devirtualization and bitset lowering.
+  PM.add(createGlobalDCEPass());
+
+  // Apply whole-program devirtualization and virtual constant propagation.
+  PM.add(createWholeProgramDevirtPass());
+}
+
 void PassManagerBuilder::addLateLTOOptimizationPasses(
     legacy::PassManagerBase &PM) {
   // Delete basic blocks, which optimization passes may have killed.
@@ -675,6 +685,9 @@ void PassManagerBuilder::populateLTOPassManager(legacy::PassManagerBase &PM) {
   if (VerifyInput)
     PM.add(createVerifierPass());
 
+  if (OptLevel != 0)
+    addEarlyLTOOptimizationPasses(PM);
+
   if (OptLevel > 1)
     addLTOOptimizationPasses(PM);