[clangd] Log memory usage of DexIndex and MemIndex

This patch prints information about built index size estimation to
verbose logs. This is useful for optimizing memory usage of DexIndex and
comparisons with MemIndex.

Reviewed by: sammccall

Differential Revision: https://reviews.llvm.org/D51154

llvm-svn: 340601

clang-tools-extra/clangd/index/FileIndex.cpp

@@ -118,5 +118,9 @@ void FileIndex::findOccurrences(
   log("findOccurrences is not implemented.");
 }
 
+size_t FileIndex::estimateMemoryUsage() const {
+  return Index.estimateMemoryUsage();
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/index/FileIndex.h

@@ -81,6 +81,9 @@ class FileIndex : public SymbolIndex {
   void findOccurrences(const OccurrencesRequest &Req,
                        llvm::function_ref<void(const SymbolOccurrence &)>
                            Callback) const override;
+
+  size_t estimateMemoryUsage() const override;
+
 private:
   FileSymbols FSymbols;
   MemIndex Index;

clang-tools-extra/clangd/index/Index.h

@@ -385,6 +385,12 @@ class SymbolIndex {
   virtual void findOccurrences(
       const OccurrencesRequest &Req,
       llvm::function_ref<void(const SymbolOccurrence &)> Callback) const = 0;
+
+  /// Returns estimated size of index (in bytes).
+  // FIXME(kbobyrev): Currently, this only returns the size of index itself
+  // excluding the size of actual symbol slab index refers to. We should include
+  // both.
+  virtual size_t estimateMemoryUsage() const = 0;
 };
 
 } // namespace clangd

clang-tools-extra/clangd/index/MemIndex.cpp

@@ -26,6 +26,9 @@ void MemIndex::build(std::shared_ptr<std::vector<const Symbol *>> Syms) {
     Index = std::move(TempIndex);
     Symbols = std::move(Syms); // Relase old symbols.
   }
+
+  vlog("Built MemIndex with estimated memory usage {0} bytes.",
+       estimateMemoryUsage());
 }
 
 std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
@@ -98,5 +101,10 @@ getSymbolsFromSlab(SymbolSlab Slab) {
                                                       &Snap->Pointers);
 }
 
+size_t MemIndex::estimateMemoryUsage() const {
+  std::lock_guard<std::mutex> Lock(Mutex);
+  return Index.getMemorySize();
+}
+
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/index/MemIndex.h

@@ -39,7 +39,10 @@ class MemIndex : public SymbolIndex {
                        llvm::function_ref<void(const SymbolOccurrence &)>
                            Callback) const override;
 
+  size_t estimateMemoryUsage() const override;
+
 private:
+
   std::shared_ptr<std::vector<const Symbol *>> Symbols;
   // Index is a set of symbols that are deduplicated by symbol IDs.
   // FIXME: build smarter index structure.

clang-tools-extra/clangd/index/Merge.cpp

@@ -84,6 +84,10 @@ class MergedIndex : public SymbolIndex {
     log("findOccurrences is not implemented.");
   }
 
+  size_t estimateMemoryUsage() const override {
+    return Dynamic->estimateMemoryUsage() + Static->estimateMemoryUsage();
+  }
+
 private:
   const SymbolIndex *Dynamic, *Static;
 };

clang-tools-extra/clangd/index/dex/DexIndex.cpp

@@ -67,6 +67,9 @@ void DexIndex::build(std::shared_ptr<std::vector<const Symbol *>> Syms) {
     InvertedIndex = std::move(TempInvertedIndex);
     SymbolQuality = std::move(TempSymbolQuality);
   }
+
+  vlog("Built DexIndex with estimated memory usage {0} bytes.",
+       estimateMemoryUsage());
 }
 
 std::unique_ptr<SymbolIndex> DexIndex::build(SymbolSlab Slab) {
@@ -171,6 +174,20 @@ void DexIndex::findOccurrences(
   log("findOccurrences is not implemented.");
 }
 
+size_t DexIndex::estimateMemoryUsage() const {
+  std::lock_guard<std::mutex> Lock(Mutex);
+
+  size_t Bytes =
+      LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol *>);
+  Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>);
+  Bytes += InvertedIndex.size() * sizeof(Token);
+
+  for (const auto &P : InvertedIndex) {
+    Bytes += P.second.size() * sizeof(DocID);
+  }
+  return Bytes;
+}
+
 } // namespace dex
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/index/dex/DexIndex.h

@@ -57,7 +57,10 @@ class DexIndex : public SymbolIndex {
                        llvm::function_ref<void(const SymbolOccurrence &)>
                            Callback) const override;
 
+  size_t estimateMemoryUsage() const override;
+
 private:
+
   mutable std::mutex Mutex;
 
   std::shared_ptr<std::vector<const Symbol *>> Symbols /*GUARDED_BY(Mutex)*/;

clang-tools-extra/unittests/clangd/CodeCompleteTests.cpp

@@ -923,6 +923,10 @@ class IndexRequestCollector : public SymbolIndex {
                        llvm::function_ref<void(const SymbolOccurrence &)>
                            Callback) const override {}
 
+  // This is incorrect, but IndexRequestCollector is not an actual index and it
+  // isn't used in production code.
+  size_t estimateMemoryUsage() const override { return 0; }
+
   const std::vector<FuzzyFindRequest> allRequests() const { return Requests; }
 
 private: