Introduce FileEntryRef and use it when handling includes to report co…

…rrect dependencies

when the FileManager is reused across invocations

This commit introduces a parallel API to FileManager's getFile: getFileEntryRef, which returns
a reference to the FileEntry, and the name that was used to access the file. In the case of
a VFS with 'use-external-names', the FileEntyRef contains the external name of the file,
not the filename that was used to access it.

The new API is adopted only in the HeaderSearch and Preprocessor for include file lookup, so that the
accessed path can be propagated to SourceManager's FileInfo. SourceManager's FileInfo now can report this accessed path, using
the new getName method. This API is then adopted in the dependency collector, which now correctly reports dependencies when a file
is included both using a symlink and a real path in the case when the FileManager is reused across multiple Preprocessor invocations.

Note that this patch does not fix all dependency collector issues, as the same problem is still present in other cases when dependencies
are obtained using FileSkipped, InclusionDirective, and HasInclude. This will be fixed in follow-up commits.

Differential Revision: https://reviews.llvm.org/D65907

llvm-svn: 369680

clang/include/clang/Basic/FileManager.h

@@ -106,6 +106,32 @@ class FileEntry {
   bool isOpenForTests() const { return File != nullptr; }
 };
 
+/// A reference to a \c FileEntry that includes the name of the file as it was
+/// accessed by the FileManager's client.
+class FileEntryRef {
+public:
+  FileEntryRef(StringRef Name, const FileEntry &Entry)
+      : Name(Name), Entry(Entry) {}
+
+  const StringRef getName() const { return Name; }
+
+  const FileEntry &getFileEntry() const { return Entry; }
+
+  off_t getSize() const { return Entry.getSize(); }
+
+  unsigned getUID() const { return Entry.getUID(); }
+
+  const llvm::sys::fs::UniqueID &getUniqueID() const {
+    return Entry.getUniqueID();
+  }
+
+  time_t getModificationTime() const { return Entry.getModificationTime(); }
+
+private:
+  StringRef Name;
+  const FileEntry &Entry;
+};
+
 /// Implements support for file system lookup, file system caching,
 /// and directory search management.
 ///
@@ -143,13 +169,25 @@ class FileManager : public RefCountedBase<FileManager> {
   llvm::StringMap<llvm::ErrorOr<DirectoryEntry &>, llvm::BumpPtrAllocator>
   SeenDirEntries;
 
+  /// A reference to the file entry that is associated with a particular
+  /// filename, or a reference to another filename that should be looked up
+  /// instead of the accessed filename.
+  ///
+  /// The reference to another filename is specifically useful for Redirecting
+  /// VFSs that use external names. In that case, the \c FileEntryRef returned
+  /// by the \c FileManager will have the external name, and not the name that
+  /// was used to lookup the file.
+  using SeenFileEntryOrRedirect =
+      llvm::PointerUnion<FileEntry *, const StringRef *>;
+
   /// A cache that maps paths to file entries (either real or
   /// virtual) we have looked up, or an error that occurred when we looked up
   /// the file.
   ///
   /// \see SeenDirEntries
-  llvm::StringMap<llvm::ErrorOr<FileEntry &>, llvm::BumpPtrAllocator>
-  SeenFileEntries;
+  llvm::StringMap<llvm::ErrorOr<SeenFileEntryOrRedirect>,
+                  llvm::BumpPtrAllocator>
+      SeenFileEntries;
 
   /// The canonical names of directories.
   llvm::DenseMap<const DirectoryEntry *, llvm::StringRef> CanonicalDirNames;
@@ -200,6 +238,9 @@ class FileManager : public RefCountedBase<FileManager> {
   /// Removes the FileSystemStatCache object from the manager.
   void clearStatCache();
 
+  /// Returns the number of unique real file entries cached by the file manager.
+  size_t getNumUniqueRealFiles() const { return UniqueRealFiles.size(); }
+
   /// Lookup, cache, and verify the specified directory (real or
   /// virtual).
   ///
@@ -215,6 +256,10 @@ class FileManager : public RefCountedBase<FileManager> {
   /// Lookup, cache, and verify the specified file (real or
   /// virtual).
   ///
+  /// This function is deprecated and will be removed at some point in the
+  /// future, new clients should use
+  ///  \c getFileRef.
+  ///
   /// This returns a \c std::error_code if there was an error loading the file.
   /// If there is no error, the FileEntry is guaranteed to be non-NULL.
   ///
@@ -225,6 +270,24 @@ class FileManager : public RefCountedBase<FileManager> {
   llvm::ErrorOr<const FileEntry *>
   getFile(StringRef Filename, bool OpenFile = false, bool CacheFailure = true);
 
+  /// Lookup, cache, and verify the specified file (real or virtual). Return the
+  /// reference to the file entry together with the exact path that was used to
+  /// access a file by a particular call to getFileRef. If the underlying VFS is
+  /// a redirecting VFS that uses external file names, the returned FileEntryRef
+  /// will use the external name instead of the filename that was passed to this
+  /// method.
+  ///
+  /// This returns a \c std::error_code if there was an error loading the file,
+  /// or a \c FileEntryRef otherwise.
+  ///
+  /// \param OpenFile if true and the file exists, it will be opened.
+  ///
+  /// \param CacheFailure If true and the file does not exist, we'll cache
+  /// the failure to find this file.
+  llvm::ErrorOr<FileEntryRef> getFileRef(StringRef Filename,
+                                         bool OpenFile = false,
+                                         bool CacheFailure = true);
+
   /// Returns the current file system options
   FileSystemOptions &getFileSystemOpts() { return FileSystemOpts; }
   const FileSystemOptions &getFileSystemOpts() const { return FileSystemOpts; }

clang/include/clang/Basic/SourceManager.h

@@ -265,16 +265,21 @@ namespace SrcMgr {
     llvm::PointerIntPair<const ContentCache*, 3, CharacteristicKind>
         ContentAndKind;
 
+    /// The filename that is used to access the file entry represented by the
+    /// content cache.
+    StringRef Filename;
+
   public:
     /// Return a FileInfo object.
     static FileInfo get(SourceLocation IL, const ContentCache *Con,
-                        CharacteristicKind FileCharacter) {
+                        CharacteristicKind FileCharacter, StringRef Filename) {
       FileInfo X;
       X.IncludeLoc = IL.getRawEncoding();
       X.NumCreatedFIDs = 0;
       X.HasLineDirectives = false;
       X.ContentAndKind.setPointer(Con);
       X.ContentAndKind.setInt(FileCharacter);
+      X.Filename = Filename;
       return X;
     }
 
@@ -299,6 +304,10 @@ namespace SrcMgr {
     void setHasLineDirectives() {
       HasLineDirectives = true;
     }
+
+    /// Returns the name of the file that was used when the file was loaded from
+    /// the underlying file system.
+    StringRef getName() const { return Filename; }
   };
 
   /// Each ExpansionInfo encodes the expansion location - where
@@ -821,7 +830,18 @@ class SourceManager : public RefCountedBase<SourceManager> {
     const SrcMgr::ContentCache *IR =
         getOrCreateContentCache(SourceFile, isSystem(FileCharacter));
     assert(IR && "getOrCreateContentCache() cannot return NULL");
-    return createFileID(IR, IncludePos, FileCharacter, LoadedID, LoadedOffset);
+    return createFileID(IR, SourceFile->getName(), IncludePos, FileCharacter,
+                        LoadedID, LoadedOffset);
+  }
+
+  FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
+                      SrcMgr::CharacteristicKind FileCharacter,
+                      int LoadedID = 0, unsigned LoadedOffset = 0) {
+    const SrcMgr::ContentCache *IR = getOrCreateContentCache(
+        &SourceFile.getFileEntry(), isSystem(FileCharacter));
+    assert(IR && "getOrCreateContentCache() cannot return NULL");
+    return createFileID(IR, SourceFile.getName(), IncludePos, FileCharacter,
+                        LoadedID, LoadedOffset);
   }
 
   /// Create a new FileID that represents the specified memory buffer.
@@ -832,9 +852,10 @@ class SourceManager : public RefCountedBase<SourceManager> {
                       SrcMgr::CharacteristicKind FileCharacter = SrcMgr::C_User,
                       int LoadedID = 0, unsigned LoadedOffset = 0,
                       SourceLocation IncludeLoc = SourceLocation()) {
+    StringRef Name = Buffer->getBufferIdentifier();
     return createFileID(
         createMemBufferContentCache(Buffer.release(), /*DoNotFree*/ false),
-        IncludeLoc, FileCharacter, LoadedID, LoadedOffset);
+        Name, IncludeLoc, FileCharacter, LoadedID, LoadedOffset);
   }
 
   enum UnownedTag { Unowned };
@@ -847,8 +868,9 @@ class SourceManager : public RefCountedBase<SourceManager> {
                       SrcMgr::CharacteristicKind FileCharacter = SrcMgr::C_User,
                       int LoadedID = 0, unsigned LoadedOffset = 0,
                       SourceLocation IncludeLoc = SourceLocation()) {
-    return createFileID(createMemBufferContentCache(Buffer, /*DoNotFree*/true),
-                        IncludeLoc, FileCharacter, LoadedID, LoadedOffset);
+    return createFileID(createMemBufferContentCache(Buffer, /*DoNotFree*/ true),
+                        Buffer->getBufferIdentifier(), IncludeLoc,
+                        FileCharacter, LoadedID, LoadedOffset);
   }
 
   /// Get the FileID for \p SourceFile if it exists. Otherwise, create a
@@ -997,6 +1019,19 @@ class SourceManager : public RefCountedBase<SourceManager> {
     return Content->OrigEntry;
   }
 
+  /// Returns the FileEntryRef for the provided FileID.
+  Optional<FileEntryRef> getFileEntryRefForID(FileID FID) const {
+    bool Invalid = false;
+    const SrcMgr::SLocEntry &Entry = getSLocEntry(FID, &Invalid);
+    if (Invalid || !Entry.isFile())
+      return None;
+
+    const SrcMgr::ContentCache *Content = Entry.getFile().getContentCache();
+    if (!Content || !Content->OrigEntry)
+      return None;
+    return FileEntryRef(Entry.getFile().getName(), *Content->OrigEntry);
+  }
+
   /// Returns the FileEntry record for the provided SLocEntry.
   const FileEntry *getFileEntryForSLocEntry(const SrcMgr::SLocEntry &sloc) const
   {
@@ -1785,10 +1820,10 @@ class SourceManager : public RefCountedBase<SourceManager> {
   ///
   /// This works regardless of whether the ContentCache corresponds to a
   /// file or some other input source.
-  FileID createFileID(const SrcMgr::ContentCache* File,
+  FileID createFileID(const SrcMgr::ContentCache *File, StringRef Filename,
                       SourceLocation IncludePos,
-                      SrcMgr::CharacteristicKind DirCharacter,
-                      int LoadedID, unsigned LoadedOffset);
+                      SrcMgr::CharacteristicKind DirCharacter, int LoadedID,
+                      unsigned LoadedOffset);
 
   const SrcMgr::ContentCache *
     getOrCreateContentCache(const FileEntry *SourceFile,

clang/include/clang/Lex/DirectoryLookup.h

@@ -176,27 +176,20 @@ class DirectoryLookup {
   /// \param [out] MappedName if this is a headermap which maps the filename to
   /// a framework include ("Foo.h" -> "Foo/Foo.h"), set the new name to this
   /// vector and point Filename to it.
-  const FileEntry *LookupFile(StringRef &Filename, HeaderSearch &HS,
-                              SourceLocation IncludeLoc,
-                              SmallVectorImpl<char> *SearchPath,
-                              SmallVectorImpl<char> *RelativePath,
-                              Module *RequestingModule,
-                              ModuleMap::KnownHeader *SuggestedModule,
-                              bool &InUserSpecifiedSystemFramework,
-                              bool &IsFrameworkFound,
-                              bool &HasBeenMapped,
-                              SmallVectorImpl<char> &MappedName) const;
+  Optional<FileEntryRef>
+  LookupFile(StringRef &Filename, HeaderSearch &HS, SourceLocation IncludeLoc,
+             SmallVectorImpl<char> *SearchPath,
+             SmallVectorImpl<char> *RelativePath, Module *RequestingModule,
+             ModuleMap::KnownHeader *SuggestedModule,
+             bool &InUserSpecifiedSystemFramework, bool &IsFrameworkFound,
+             bool &HasBeenMapped, SmallVectorImpl<char> &MappedName) const;
 
 private:
-  const FileEntry *DoFrameworkLookup(
-      StringRef Filename, HeaderSearch &HS,
-      SmallVectorImpl<char> *SearchPath,
-      SmallVectorImpl<char> *RelativePath,
-      Module *RequestingModule,
+  Optional<FileEntryRef> DoFrameworkLookup(
+      StringRef Filename, HeaderSearch &HS, SmallVectorImpl<char> *SearchPath,
+      SmallVectorImpl<char> *RelativePath, Module *RequestingModule,
       ModuleMap::KnownHeader *SuggestedModule,
-      bool &InUserSpecifiedSystemFramework,
-      bool &IsFrameworkFound) const;
-
+      bool &InUserSpecifiedSystemFramework, bool &IsFrameworkFound) const;
 };
 
 }  // end namespace clang

clang/include/clang/Lex/HeaderMap.h

@@ -13,6 +13,7 @@
 #ifndef LLVM_CLANG_LEX_HEADERMAP_H
 #define LLVM_CLANG_LEX_HEADERMAP_H
 
+#include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/Support/Compiler.h"
@@ -21,8 +22,6 @@
 
 namespace clang {
 
-class FileEntry;
-class FileManager;
 struct HMapBucket;
 struct HMapHeader;
 
@@ -78,7 +77,7 @@ class HeaderMap : private HeaderMapImpl {
   /// NULL and the file is found, RawPath will be set to the raw path at which
   /// the file was found in the file system. For example, for a search path
   /// ".." and a filename "../file.h" this would be "../../file.h".
-  const FileEntry *LookupFile(StringRef Filename, FileManager &FM) const;
+  Optional<FileEntryRef> LookupFile(StringRef Filename, FileManager &FM) const;
 
   using HeaderMapImpl::lookupFilename;
   using HeaderMapImpl::getFileName;

clang/include/clang/Lex/HeaderSearch.h

@@ -395,7 +395,7 @@ class HeaderSearch {
   /// found in any of searched SearchDirs. Will be set to false if a framework
   /// is found only through header maps. Doesn't guarantee the requested file is
   /// found.
-  const FileEntry *LookupFile(
+  Optional<FileEntryRef> LookupFile(
       StringRef Filename, SourceLocation IncludeLoc, bool isAngled,
       const DirectoryLookup *FromDir, const DirectoryLookup *&CurDir,
       ArrayRef<std::pair<const FileEntry *, const DirectoryEntry *>> Includers,
@@ -410,7 +410,7 @@ class HeaderSearch {
   /// within ".../Carbon.framework/Headers/Carbon.h", check to see if
   /// HIToolbox is a subframework within Carbon.framework.  If so, return
   /// the FileEntry for the designated file, otherwise return null.
-  const FileEntry *LookupSubframeworkHeader(
+  Optional<FileEntryRef> LookupSubframeworkHeader(
       StringRef Filename, const FileEntry *ContextFileEnt,
       SmallVectorImpl<char> *SearchPath, SmallVectorImpl<char> *RelativePath,
       Module *RequestingModule, ModuleMap::KnownHeader *SuggestedModule);
@@ -649,7 +649,7 @@ class HeaderSearch {
 
   /// Look up the file with the specified name and determine its owning
   /// module.
-  const FileEntry *
+  Optional<FileEntryRef>
   getFileAndSuggestModule(StringRef FileName, SourceLocation IncludeLoc,
                           const DirectoryEntry *Dir, bool IsSystemHeaderDir,
                           Module *RequestingModule,

clang/include/clang/Lex/Preprocessor.h

@@ -1949,17 +1949,15 @@ class Preprocessor {
 
   /// Given a "foo" or \<foo> reference, look up the indicated file.
   ///
-  /// Returns null on failure.  \p isAngled indicates whether the file
+  /// Returns None on failure.  \p isAngled indicates whether the file
   /// reference is for system \#include's or not (i.e. using <> instead of "").
-  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
-                              bool isAngled, const DirectoryLookup *FromDir,
-                              const FileEntry *FromFile,
-                              const DirectoryLookup *&CurDir,
-                              SmallVectorImpl<char> *SearchPath,
-                              SmallVectorImpl<char> *RelativePath,
-                              ModuleMap::KnownHeader *SuggestedModule,
-                              bool *IsMapped, bool *IsFrameworkFound,
-                              bool SkipCache = false);
+  Optional<FileEntryRef>
+  LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled,
+             const DirectoryLookup *FromDir, const FileEntry *FromFile,
+             const DirectoryLookup *&CurDir, SmallVectorImpl<char> *SearchPath,
+             SmallVectorImpl<char> *RelativePath,
+             ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped,
+             bool *IsFrameworkFound, bool SkipCache = false);
 
   /// Get the DirectoryLookup structure used to find the current
   /// FileEntry, if CurLexer is non-null and if applicable.
@@ -2202,6 +2200,15 @@ class Preprocessor {
     }
   };
 
+  Optional<FileEntryRef> LookupHeaderIncludeOrImport(
+      const DirectoryLookup *&CurDir, StringRef Filename,
+      SourceLocation FilenameLoc, CharSourceRange FilenameRange,
+      const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
+      bool &IsMapped, const DirectoryLookup *LookupFrom,
+      const FileEntry *LookupFromFile, SmallString<128> &NormalizedPath,
+      SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
+      ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
+
   // File inclusion.
   void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok,
                               const DirectoryLookup *LookupFrom = nullptr,