Introduce a new data structure, LazyVector, which is a vector whose

contents are lazily loaded on demand from an external source (e.g., an
ExternalASTSource or ExternalSemaSource). The "loaded" entities are
kept separate from the "local" entities, so that the two can grow
independently.

Switch Sema::TentativeDefinitions from a normal vector that is eagerly
populated by the ASTReader into one of these LazyVectors, making the
ASTReader a bit more like me (i.e., lazy).



git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@136262 91177308-0d34-0410-b5e6-96231b3b80d8

include/clang/AST/ExternalASTSource.h

@@ -300,6 +300,162 @@ struct LazyOffsetPtr {
   }
 };
 
+/// \brief Represents a lazily-loaded vector of data.
+///
+/// The lazily-loaded vector of data contains data that is partially loaded
+/// from an external source and partially added by local translation. The 
+/// items loaded from the external source are loaded lazily, when needed for
+/// iteration over the complete vector.
+template<typename T, typename Source, 
+         void (Source::*Loader)(SmallVectorImpl<T>&),
+         unsigned LoadedStorage = 2, unsigned LocalStorage = 4>
+class LazyVector {
+  SmallVector<T, LoadedStorage> Loaded;
+  SmallVector<T, LocalStorage> Local;
+
+public:
+  // Iteration over the elements in the vector.
+  class iterator {
+    LazyVector *Self;
+
+    /// \brief Position within the vector..
+    ///
+    /// In a complete iteration, the Position field walks the range [-M, N),
+    /// where negative values are used to indicate elements
+    /// loaded from the external source while non-negative values are used to
+    /// indicate elements added via \c push_back().
+    /// However, to provide iteration in source order (for, e.g., chained
+    /// precompiled headers), dereferencing the iterator flips the negative
+    /// values (corresponding to loaded entities), so that position -M 
+    /// corresponds to element 0 in the loaded entities vector, position -M+1
+    /// corresponds to element 1 in the loaded entities vector, etc. This
+    /// gives us a reasonably efficient, source-order walk.
+    int Position;
+
+  public:
+    typedef T                   value_type;
+    typedef value_type&         reference;
+    typedef value_type*         pointer;
+    typedef std::random_access_iterator_tag iterator_category;
+    typedef int                 difference_type;
+
+    iterator() : Self(0), Position(0) { }
+
+    iterator(LazyVector *Self, int Position) 
+      : Self(Self), Position(Position) { }
+
+    reference operator*() const {
+      if (Position < 0)
+        return Self->Loaded.end()[Position];
+      return Self->Local[Position];
+    }
+
+    pointer operator->() const {
+      if (Position < 0)
+        return &Self->Loaded.end()[Position];
+
+      return &Self->Local[Position];        
+    }
+
+    reference operator[](difference_type D) {
+      return *(*this + D);
+    }
+
+    iterator &operator++() {
+      ++Position;
+      return *this;
+    }
+
+    iterator operator++(int) {
+      iterator Prev(*this);
+      ++Position;
+      return Prev;
+    }
+
+    iterator &operator--() {
+      --Position;
+      return *this;
+    }
+
+    iterator operator--(int) {
+      iterator Prev(*this);
+      --Position;
+      return Prev;
+    }
+
+    friend bool operator==(const iterator &X, const iterator &Y) {
+      return X.Position == Y.Position;
+    }
+
+    friend bool operator!=(const iterator &X, const iterator &Y) {
+      return X.Position != Y.Position;
+    }
+
+    friend bool operator<(const iterator &X, const iterator &Y) {
+      return X.Position < Y.Position;
+    }
+
+    friend bool operator>(const iterator &X, const iterator &Y) {
+      return X.Position > Y.Position;
+    }
+
+    friend bool operator<=(const iterator &X, const iterator &Y) {
+      return X.Position < Y.Position;
+    }
+
+    friend bool operator>=(const iterator &X, const iterator &Y) {
+      return X.Position > Y.Position;
+    }
+
+    friend iterator& operator+=(iterator &X, difference_type D) {
+      X.Position += D;
+      return X;
+    }
+
+    friend iterator& operator-=(iterator &X, difference_type D) {
+      X.Position -= D;
+      return X;
+    }
+
+    friend iterator operator+(iterator X, difference_type D) {
+      X.Position += D;
+      return X;
+    }
+
+    friend iterator operator+(difference_type D, iterator X) {
+      X.Position += D;
+      return X;
+    }
+
+    friend difference_type operator-(const iterator &X, const iterator &Y) {
+      return X.Position - Y.Position;
+    }
+
+    friend iterator operator-(iterator X, difference_type D) {
+      X.Position -= D;
+      return X;
+    }
+  };
+  friend class iterator;
+
+  iterator begin(Source *source, bool LocalOnly = false) {
+    if (LocalOnly)
+      return iterator(this, 0);
+
+    if (source)
+      (source->*Loader)(Loaded);
+    return iterator(this, -(int)Loaded.size());
+  }
+
+  iterator end() {
+    return iterator(this, Local.size());
+  }
+
+  void push_back(const T& LocalValue) {
+    Local.push_back(LocalValue);
+  }
+};
+
 /// \brief A lazy pointer to a statement.
 typedef LazyOffsetPtr<Stmt, uint64_t, &ExternalASTSource::GetExternalDeclStmt>
   LazyDeclStmtPtr;

include/clang/Sema/ExternalSemaSource.h

@@ -22,7 +22,8 @@ struct ObjCMethodList;
 class Sema;
 class Scope;
 class LookupResult;
-
+class VarDecl;
+
 /// \brief An abstract interface that should be implemented by
 /// external AST sources that also provide information for semantic
 /// analysis.
@@ -64,12 +65,22 @@ class ExternalSemaSource : public ExternalASTSource {
   /// \return true to tell Sema to recover using the LookupResult.
   virtual bool LookupUnqualified(LookupResult &R, Scope *S) { return false; }
 
+  /// \brief Read the set of tentative definitions know to the external Sema
+  /// source.
+  ///
+  /// The external source should append its own tentative definitions to the
+  /// given vector of tentative definitions. Note that this routine may be
+  /// invoked multiple times; the external source should take care not to
+  /// introduce the same declarations repeatedly.
+  virtual void ReadTentativeDefinitions(
+                                  SmallVectorImpl<VarDecl *> &TentativeDefs) {}
+
   // isa/cast/dyn_cast support
   static bool classof(const ExternalASTSource *Source) {
     return Source->SemaSource;
   }
   static bool classof(const ExternalSemaSource *) { return true; }
-};
+}; 
 
 } // end namespace clang
 

include/clang/Sema/Sema.h

@@ -20,6 +20,7 @@
 #include "clang/Sema/IdentifierResolver.h"
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/ExternalSemaSource.h"
 #include "clang/Sema/LocInfoType.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "clang/AST/Expr.h"
@@ -280,8 +281,12 @@ class Sema {
   ///     not visible.
   llvm::DenseMap<DeclarationName, NamedDecl *> LocallyScopedExternalDecls;
 
+  typedef LazyVector<VarDecl *, ExternalSemaSource, 
+                     &ExternalSemaSource::ReadTentativeDefinitions, 2, 2>
+    TentativeDefinitionsType;
+
   /// \brief All the tentative definitions encountered in the TU.
-  SmallVector<VarDecl *, 2> TentativeDefinitions;
+  TentativeDefinitionsType TentativeDefinitions;
 
   /// \brief The set of file scoped decls seen so far that have not been used
   /// and must warn if not used. Only contains the first declaration.

include/clang/Serialization/ASTReader.h

@@ -1371,6 +1371,9 @@ class ASTReader
   virtual void ReadKnownNamespaces(
                            SmallVectorImpl<NamespaceDecl *> &Namespaces);
 
+  virtual void ReadTentativeDefinitions(
+                 SmallVectorImpl<VarDecl *> &TentativeDefs);
+
   /// \brief Load a selector from disk, registering its ID if it exists.
   void LoadSelector(Selector Sel);
 

lib/Sema/Sema.cpp

@@ -473,8 +473,12 @@ void Sema::ActOnEndOfTranslationUnit() {
   //   identifier, with the composite type as of the end of the
   //   translation unit, with an initializer equal to 0.
   llvm::SmallSet<VarDecl *, 32> Seen;
-  for (unsigned i = 0, e = TentativeDefinitions.size(); i != e; ++i) {
-    VarDecl *VD = TentativeDefinitions[i]->getActingDefinition();
+  for (TentativeDefinitionsType::iterator 
+            T = TentativeDefinitions.begin(ExternalSource),
+         TEnd = TentativeDefinitions.end();
+       T != TEnd; ++T) 
+  {
+    VarDecl *VD = (*T)->getActingDefinition();
 
     // If the tentative definition was completed, getActingDefinition() returns
     // null. If we've already seen this variable before, insert()'s second

lib/Serialization/ASTReader.cpp

@@ -4328,13 +4328,6 @@ void ASTReader::InitializeSema(Sema &S) {
   }
   PreloadedDecls.clear();
 
-  // If there were any tentative definitions, deserialize them and add
-  // them to Sema's list of tentative definitions.
-  for (unsigned I = 0, N = TentativeDefinitions.size(); I != N; ++I) {
-    VarDecl *Var = cast<VarDecl>(GetDecl(TentativeDefinitions[I]));
-    SemaObj->TentativeDefinitions.push_back(Var);
-  }
-
   // If there were any unused file scoped decls, deserialize them and add to
   // Sema's list of unused file scoped decls.
   for (unsigned I = 0, N = UnusedFileScopedDecls.size(); I != N; ++I) {
@@ -4571,6 +4564,16 @@ void ASTReader::ReadKnownNamespaces(
   }
 }
 
+void ASTReader::ReadTentativeDefinitions(
+                  SmallVectorImpl<VarDecl *> &TentativeDefs) {
+  for (unsigned I = 0, N = TentativeDefinitions.size(); I != N; ++I) {
+    VarDecl *Var = dyn_cast_or_null<VarDecl>(GetDecl(TentativeDefinitions[I]));
+    if (Var)
+      TentativeDefs.push_back(Var);
+  }
+  TentativeDefinitions.clear();
+}
+
 void ASTReader::LoadSelector(Selector Sel) {
   // It would be complicated to avoid reading the methods anyway. So don't.
   ReadMethodPool(Sel);

lib/Serialization/ASTWriter.cpp

@@ -2805,10 +2805,13 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   // TentativeDefinitions order.  Generally, this record will be empty for
   // headers.
   RecordData TentativeDefinitions;
-  for (unsigned i = 0, e = SemaRef.TentativeDefinitions.size(); i != e; ++i) {
-    AddDeclRef(SemaRef.TentativeDefinitions[i], TentativeDefinitions);
+  for (Sema::TentativeDefinitionsType::iterator 
+            T = SemaRef.TentativeDefinitions.begin(0, true),
+         TEnd = SemaRef.TentativeDefinitions.end();
+       T != TEnd; ++T) {
+    AddDeclRef(*T, TentativeDefinitions);
   }
-
+  
   // Build a record containing all of the file scoped decls in this file.
   RecordData UnusedFileScopedDecls;
   for (unsigned i=0, e = SemaRef.UnusedFileScopedDecls.size(); i !=e; ++i)
@@ -3072,11 +3075,13 @@ void ASTWriter::WriteASTChain(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   // Build a record containing all of the new tentative definitions in this
   // file, in TentativeDefinitions order.
   RecordData TentativeDefinitions;
-  for (unsigned i = 0, e = SemaRef.TentativeDefinitions.size(); i != e; ++i) {
-    if (SemaRef.TentativeDefinitions[i]->getPCHLevel() == 0)
-      AddDeclRef(SemaRef.TentativeDefinitions[i], TentativeDefinitions);
+  for (Sema::TentativeDefinitionsType::iterator 
+       T = SemaRef.TentativeDefinitions.begin(0, true),
+       TEnd = SemaRef.TentativeDefinitions.end();
+       T != TEnd; ++T) {
+    AddDeclRef(*T, TentativeDefinitions);
   }
-
+  
   // Build a record containing all of the file scoped decls in this file.
   RecordData UnusedFileScopedDecls;
   for (unsigned i=0, e = SemaRef.UnusedFileScopedDecls.size(); i !=e; ++i) {