diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td index 11c706ebf84b5..4c4659ed517e0 100644 --- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td +++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td @@ -44,6 +44,9 @@ def err_pch_diagopt_mismatch : Error<"%0 is currently enabled, but was not in " "the PCH file">; def err_pch_modulecache_mismatch : Error<"PCH was compiled with module cache " "path '%0', but the path is currently '%1'">; +def err_pch_vfsoverlay_mismatch : Error<"PCH was compiled with different VFS overlay files than are currently in use">; +def note_pch_vfsoverlay_files : Note<"%select{PCH|current translation unit}0 has the following VFS overlays:\n%1">; +def note_pch_vfsoverlay_empty : Note<"%select{PCH|current translation unit}0 has no VFS overlays">; def err_pch_version_too_old : Error< "PCH file uses an older PCH format that is no longer supported">; diff --git a/clang/include/clang/Basic/FileManager.h b/clang/include/clang/Basic/FileManager.h index 56cb093dd8c37..997c17a0ffcfc 100644 --- a/clang/include/clang/Basic/FileManager.h +++ b/clang/include/clang/Basic/FileManager.h @@ -248,6 +248,10 @@ class FileManager : public RefCountedBase { return FS; } + /// Enable or disable tracking of VFS usage. Used to not track full header + /// search and implicit modulemap lookup. + void trackVFSUsage(bool Active); + void setVirtualFileSystem(IntrusiveRefCntPtr FS) { this->FS = std::move(FS); } diff --git a/clang/include/clang/Lex/HeaderSearch.h b/clang/include/clang/Lex/HeaderSearch.h index a2c33842924b1..705dcfa8aacc3 100644 --- a/clang/include/clang/Lex/HeaderSearch.h +++ b/clang/include/clang/Lex/HeaderSearch.h @@ -576,6 +576,13 @@ class HeaderSearch { /// Note: implicit module maps don't contribute to entry usage. std::vector computeUserEntryUsage() const; + /// Collect which HeaderSearchOptions::VFSOverlayFiles have been meaningfully + /// used so far and mark their index with 'true' in the resulting bit vector. + /// + /// Note: this ignores VFSs that redirect non-affecting files such as unused + /// modulemaps. + std::vector collectVFSUsageAndClear() const; + /// This method returns a HeaderMap for the specified /// FileEntry, uniquing them through the 'HeaderMaps' datastructure. const HeaderMap *CreateHeaderMap(FileEntryRef FE); diff --git a/clang/include/clang/Lex/HeaderSearchOptions.h b/clang/include/clang/Lex/HeaderSearchOptions.h index fa2d0b502d72c..637dc77e5d957 100644 --- a/clang/include/clang/Lex/HeaderSearchOptions.h +++ b/clang/include/clang/Lex/HeaderSearchOptions.h @@ -263,6 +263,10 @@ class HeaderSearchOptions { LLVM_PREFERRED_TYPE(bool) unsigned ModulesStrictContextHash : 1; + /// Whether to include ivfsoverlay usage information in written AST files. + LLVM_PREFERRED_TYPE(bool) + unsigned ModulesIncludeVFSUsage : 1; + HeaderSearchOptions(StringRef _Sysroot = "/") : Sysroot(_Sysroot), ModuleFormat("raw"), DisableModuleHash(false), ImplicitModuleMaps(false), ModuleMapFileHomeIsCwd(false), @@ -277,7 +281,7 @@ class HeaderSearchOptions { ModulesSkipDiagnosticOptions(false), ModulesSkipHeaderSearchPaths(false), ModulesSkipPragmaDiagnosticMappings(false), ModulesHashContent(false), - ModulesStrictContextHash(false) {} + ModulesStrictContextHash(false), ModulesIncludeVFSUsage(false) {} /// AddPath - Add the \p Path path to the specified \p Group list. void AddPath(StringRef Path, frontend::IncludeDirGroup Group, diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index ea084b328d612..9de925163599d 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -405,6 +405,9 @@ enum UnhashedControlBlockRecordTypes { /// Record code for the indices of used header search entries. HEADER_SEARCH_ENTRY_USAGE, + + /// Record code for the indices of used VFSs. + VFS_USAGE, }; /// Record code for extension blocks. diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index ba06ab0cd4509..7328c4acdce07 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -1780,12 +1780,13 @@ class ASTReader /// Read the control block for the named AST file. /// /// \returns true if an error occurred, false otherwise. - static bool readASTFileControlBlock(StringRef Filename, FileManager &FileMgr, - const InMemoryModuleCache &ModuleCache, - const PCHContainerReader &PCHContainerRdr, - bool FindModuleFileExtensions, - ASTReaderListener &Listener, - bool ValidateDiagnosticOptions); + static bool readASTFileControlBlock( + StringRef Filename, FileManager &FileMgr, + const InMemoryModuleCache &ModuleCache, + const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions, + ASTReaderListener &Listener, bool ValidateDiagnosticOptions, + unsigned ClientLoadCapabilities = ARR_ConfigurationMismatch | + ARR_OutOfDate); /// Determine whether the given AST file is acceptable to load into a /// translation unit with the given language and target options. @@ -2270,6 +2271,9 @@ class ASTReader SourceRange ReadSourceRange(ModuleFile &F, const RecordData &Record, unsigned &Idx, LocSeq *Seq = nullptr); + static llvm::BitVector ReadBitVector(const RecordData &Record, + const StringRef Blob); + // Read a string static std::string ReadString(const RecordDataImpl &Record, unsigned &Idx); diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index de69f99003d82..5e2f305b294ca 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -467,10 +467,10 @@ class ASTWriter : public ASTDeserializationListener, std::vector NonAffectingRanges; std::vector NonAffectingOffsetAdjustments; - /// Collects input files that didn't affect compilation of the current module, + /// Computes input files that didn't affect compilation of the current module, /// and initializes data structures necessary for leaving those files out /// during \c SourceManager serialization. - void collectNonAffectingInputFiles(); + void computeNonAffectingInputFiles(); /// Returns an adjusted \c FileID, accounting for any non-affecting input /// files. diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 9a14129d72ff3..bc0aa89966c2b 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -189,6 +189,9 @@ class ModuleFile { /// The bit vector denoting usage of each header search entry (true = used). llvm::BitVector SearchPathUsage; + /// The bit vector denoting usage of each VFS entry (true = used). + llvm::BitVector VFSUsage; + /// Whether this module has been directly imported by the /// user. bool DirectlyImported = false; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index 9a2aea5d6efa1..846fdc7253977 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -280,8 +280,12 @@ class EntryRef { /// This is not a thread safe VFS. A single instance is meant to be used only in /// one thread. Multiple instances are allowed to service multiple threads /// running in parallel. -class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem { +class DependencyScanningWorkerFilesystem + : public llvm::RTTIExtends { public: + static const char ID; + DependencyScanningWorkerFilesystem( DependencyScanningFilesystemSharedCache &SharedCache, IntrusiveRefCntPtr FS); diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h index dcdf1c171f6d7..4f9867262a275 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -45,6 +45,9 @@ enum class ScanningOutputFormat { P1689, }; +#define DSS_LAST_BITMASK_ENUM(Id) \ + LLVM_MARK_AS_BITMASK_ENUM(Id), All = llvm::NextPowerOf2(Id) - 1 + enum class ScanningOptimizations { None = 0, @@ -54,11 +57,15 @@ enum class ScanningOptimizations { /// Remove warnings from system modules. SystemWarnings = 2, - LLVM_MARK_AS_BITMASK_ENUM(SystemWarnings), - All = HeaderSearch | SystemWarnings, + /// Remove unused -ivfsoverlay arguments. + VFS = 4, + + DSS_LAST_BITMASK_ENUM(VFS), Default = All }; +#undef DSS_LAST_BITMASK_ENUM + /// The dependency scanning service contains shared configuration and state that /// is used by the individual dependency scanning workers. class DependencyScanningService { diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp index 4e77b178a86b9..6097a27e429d6 100644 --- a/clang/lib/Basic/FileManager.cpp +++ b/clang/lib/Basic/FileManager.cpp @@ -363,6 +363,13 @@ llvm::Expected FileManager::getSTDIN() { return *STDIN; } +void FileManager::trackVFSUsage(bool Active) { + FS->visit([Active](llvm::vfs::FileSystem &FileSys) { + if (auto *RFS = dyn_cast(&FileSys)) + RFS->setUsageTrackingActive(Active); + }); +} + const FileEntry *FileManager::getVirtualFile(StringRef Filename, off_t Size, time_t ModificationTime) { return &getVirtualFileRef(Filename, Size, ModificationTime).getFileEntry(); diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index feb4de2084b83..8d7b75b56d612 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -4763,6 +4763,7 @@ std::string CompilerInvocation::getModuleHash() const { if (hsOpts.ModulesStrictContextHash) { HBuilder.addRange(hsOpts.SystemHeaderPrefixes); HBuilder.addRange(hsOpts.UserEntries); + HBuilder.addRange(hsOpts.VFSOverlayFiles); const DiagnosticOptions &diagOpts = getDiagnosticOpts(); #define DIAGOPT(Name, Bits, Default) HBuilder.add(diagOpts.Name); diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp index 0f1090187734f..fcc2b56df166b 100644 --- a/clang/lib/Lex/HeaderSearch.cpp +++ b/clang/lib/Lex/HeaderSearch.cpp @@ -141,6 +141,28 @@ std::vector HeaderSearch::computeUserEntryUsage() const { return UserEntryUsage; } +std::vector HeaderSearch::collectVFSUsageAndClear() const { + std::vector VFSUsage; + if (!getHeaderSearchOpts().ModulesIncludeVFSUsage) + return VFSUsage; + + llvm::vfs::FileSystem &RootFS = FileMgr.getVirtualFileSystem(); + // TODO: This only works if the `RedirectingFileSystem`s were all created by + // `createVFSFromOverlayFiles`. + RootFS.visit([&](llvm::vfs::FileSystem &FS) { + if (auto *RFS = dyn_cast(&FS)) { + VFSUsage.push_back(RFS->hasBeenUsed()); + RFS->clearHasBeenUsed(); + } + }); + assert(VFSUsage.size() == getHeaderSearchOpts().VFSOverlayFiles.size() && + "A different number of RedirectingFileSystem's were present than " + "-ivfsoverlay options passed to Clang!"); + // VFS visit order is the opposite of VFSOverlayFiles order. + std::reverse(VFSUsage.begin(), VFSUsage.end()); + return VFSUsage; +} + /// CreateHeaderMap - This method returns a HeaderMap for the specified /// FileEntry, uniquing them through the 'HeaderMaps' datastructure. const HeaderMap *HeaderSearch::CreateHeaderMap(FileEntryRef FE) { diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index 2abe5e44e2e98..04ab42fa18826 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -4977,7 +4977,7 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl( } case HEADER_SEARCH_PATHS: { bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0; - if (!AllowCompatibleConfigurationMismatch && + if (Listener && !AllowCompatibleConfigurationMismatch && ParseHeaderSearchPaths(Record, Complain, *Listener)) Result = ConfigurationMismatch; break; @@ -4992,15 +4992,12 @@ ASTReader::ASTReadResult ASTReader::readUnhashedControlBlockImpl( Record.begin(), Record.end()); break; case HEADER_SEARCH_ENTRY_USAGE: - if (!F) - break; - unsigned Count = Record[0]; - const char *Byte = Blob.data(); - F->SearchPathUsage = llvm::BitVector(Count, false); - for (unsigned I = 0; I < Count; ++Byte) - for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I) - if (*Byte & (1 << Bit)) - F->SearchPathUsage[I] = true; + if (F) + F->SearchPathUsage = ReadBitVector(Record, Blob); + break; + case VFS_USAGE: + if (F) + F->VFSUsage = ReadBitVector(Record, Blob); break; } } @@ -5398,7 +5395,8 @@ bool ASTReader::readASTFileControlBlock( StringRef Filename, FileManager &FileMgr, const InMemoryModuleCache &ModuleCache, const PCHContainerReader &PCHContainerRdr, bool FindModuleFileExtensions, - ASTReaderListener &Listener, bool ValidateDiagnosticOptions) { + ASTReaderListener &Listener, bool ValidateDiagnosticOptions, + unsigned ClientLoadCapabilities) { // Open the AST file. std::unique_ptr OwnedBuffer; llvm::MemoryBuffer *Buffer = ModuleCache.lookupPCM(Filename); @@ -5453,7 +5451,7 @@ bool ASTReader::readASTFileControlBlock( switch (Entry.ID) { case OPTIONS_BLOCK_ID: { std::string IgnoredSuggestedPredefines; - if (ReadOptionsBlock(Stream, ARR_ConfigurationMismatch | ARR_OutOfDate, + if (ReadOptionsBlock(Stream, ClientLoadCapabilities, /*AllowCompatibleConfigurationMismatch*/ false, Listener, IgnoredSuggestedPredefines) != Success) return true; @@ -5679,7 +5677,7 @@ bool ASTReader::readASTFileControlBlock( // Scan for the UNHASHED_CONTROL_BLOCK_ID block. if (readUnhashedControlBlockImpl( - nullptr, Bytes, ARR_ConfigurationMismatch | ARR_OutOfDate, + nullptr, Bytes, ClientLoadCapabilities, /*AllowCompatibleConfigurationMismatch*/ false, &Listener, ValidateDiagnosticOptions) != Success) return true; @@ -9316,6 +9314,18 @@ SourceRange ASTReader::ReadSourceRange(ModuleFile &F, const RecordData &Record, return SourceRange(beg, end); } +llvm::BitVector ASTReader::ReadBitVector(const RecordData &Record, + const StringRef Blob) { + unsigned Count = Record[0]; + const char *Byte = Blob.data(); + llvm::BitVector Ret = llvm::BitVector(Count, false); + for (unsigned I = 0; I < Count; ++Byte) + for (unsigned Bit = 0; Bit < 8 && I < Count; ++Bit, ++I) + if (*Byte & (1 << Bit)) + Ret[I] = true; + return Ret; +} + /// Read a floating-point value llvm::APFloat ASTRecordReader::readAPFloat(const llvm::fltSemantics &Sem) { return llvm::APFloat(Sem, readAPInt()); diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index dcb18ad338932..9eefa7d1f7015 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1270,18 +1270,30 @@ void ASTWriter::writeUnhashedControlBlock(Preprocessor &PP, WritePragmaDiagnosticMappings(Diags, /* isModule = */ WritingModule); // Header search entry usage. - auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage(); - auto Abbrev = std::make_shared(); - Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE)); - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits. - Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector. - unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); { + auto HSEntryUsage = PP.getHeaderSearchInfo().computeUserEntryUsage(); + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(HEADER_SEARCH_ENTRY_USAGE)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits. + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector. + unsigned HSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); RecordData::value_type Record[] = {HEADER_SEARCH_ENTRY_USAGE, HSEntryUsage.size()}; Stream.EmitRecordWithBlob(HSUsageAbbrevCode, Record, bytes(HSEntryUsage)); } + // VFS usage. + { + auto VFSUsage = PP.getHeaderSearchInfo().collectVFSUsageAndClear(); + auto Abbrev = std::make_shared(); + Abbrev->Add(BitCodeAbbrevOp(VFS_USAGE)); + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // Number of bits. + Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Bit vector. + unsigned VFSUsageAbbrevCode = Stream.EmitAbbrev(std::move(Abbrev)); + RecordData::value_type Record[] = {VFS_USAGE, VFSUsage.size()}; + Stream.EmitRecordWithBlob(VFSUsageAbbrevCode, Record, bytes(VFSUsage)); + } + // Leave the options block. Stream.ExitBlock(); UnhashedControlBlockRange.second = Stream.GetCurrentBitNo() >> 3; @@ -4672,7 +4684,7 @@ static void AddLazyVectorDecls(ASTWriter &Writer, Vector &Vec, } } -void ASTWriter::collectNonAffectingInputFiles() { +void ASTWriter::computeNonAffectingInputFiles() { SourceManager &SrcMgr = PP->getSourceManager(); unsigned N = SrcMgr.local_sloc_entry_size(); @@ -4732,6 +4744,30 @@ void ASTWriter::collectNonAffectingInputFiles() { NonAffectingFileIDAdjustments.push_back(FileIDAdjustment); NonAffectingOffsetAdjustments.push_back(OffsetAdjustment); } + + if (!PP->getHeaderSearchInfo().getHeaderSearchOpts().ModulesIncludeVFSUsage) + return; + + FileManager &FileMgr = PP->getFileManager(); + FileMgr.trackVFSUsage(true); + // Lookup the paths in the VFS to trigger `-ivfsoverlay` usage tracking. + for (StringRef Path : + PP->getHeaderSearchInfo().getHeaderSearchOpts().VFSOverlayFiles) + FileMgr.getVirtualFileSystem().exists(Path); + for (unsigned I = 1; I != N; ++I) { + if (IsSLocAffecting[I]) { + const SrcMgr::SLocEntry *SLoc = &SrcMgr.getLocalSLocEntry(I); + if (!SLoc->isFile()) + continue; + const SrcMgr::FileInfo &File = SLoc->getFile(); + const SrcMgr::ContentCache *Cache = &File.getContentCache(); + if (!Cache->OrigEntry) + continue; + FileMgr.getVirtualFileSystem().exists( + Cache->OrigEntry->getNameAsRequested()); + } + } + FileMgr.trackVFSUsage(false); } ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, @@ -4749,7 +4785,7 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, // This needs to be done very early, since everything that writes // SourceLocations or FileIDs depends on it. - collectNonAffectingInputFiles(); + computeNonAffectingInputFiles(); writeUnhashedControlBlock(PP, Context); diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index 6f71650a3982c..1b750cec41e1c 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -194,7 +194,9 @@ static bool shouldCacheStatFailures(StringRef Filename) { DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem( DependencyScanningFilesystemSharedCache &SharedCache, IntrusiveRefCntPtr FS) - : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache), + : llvm::RTTIExtends(std::move(FS)), + SharedCache(SharedCache), WorkingDirForCacheLookup(llvm::errc::invalid_argument) { updateWorkingDirForCacheLookup(); } @@ -379,3 +381,5 @@ void DependencyScanningWorkerFilesystem::updateWorkingDirForCacheLookup() { assert(!WorkingDirForCacheLookup || llvm::sys::path::is_absolute_gnu(*WorkingDirForCacheLookup)); } + +const char DependencyScanningWorkerFilesystem::ID = 0; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 7ab4a699af6df..390cbe5aa65e1 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -9,6 +9,7 @@ #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "clang/Basic/DiagnosticDriver.h" #include "clang/Basic/DiagnosticFrontend.h" +#include "clang/Basic/DiagnosticSerialization.h" #include "clang/CodeGen/ObjectFilePCHContainerOperations.h" #include "clang/Driver/Compilation.h" #include "clang/Driver/Driver.h" @@ -59,6 +60,31 @@ class DependencyConsumerForwarder : public DependencyFileGenerator { DependencyConsumer &C; }; +static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts, + const HeaderSearchOptions &ExistingHSOpts, + DiagnosticsEngine *Diags, + const LangOptions &LangOpts) { + if (LangOpts.Modules) { + if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) { + if (Diags) { + Diags->Report(diag::err_pch_vfsoverlay_mismatch); + auto VFSNote = [&](int Type, ArrayRef VFSOverlays) { + if (VFSOverlays.empty()) { + Diags->Report(diag::note_pch_vfsoverlay_empty) << Type; + } else { + std::string Files = llvm::join(VFSOverlays, "\n"); + Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files; + } + }; + VFSNote(0, HSOpts.VFSOverlayFiles); + VFSNote(1, ExistingHSOpts.VFSOverlayFiles); + } + return true; + } + } + return false; +} + using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); /// A listener that collects the imported modules and optionally the input @@ -66,9 +92,12 @@ using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); class PrebuiltModuleListener : public ASTReaderListener { public: PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles, - llvm::SmallVector &NewModuleFiles) + llvm::SmallVector &NewModuleFiles, + const HeaderSearchOptions &HSOpts, + const LangOptions &LangOpts, DiagnosticsEngine &Diags) : PrebuiltModuleFiles(PrebuiltModuleFiles), - NewModuleFiles(NewModuleFiles) {} + NewModuleFiles(NewModuleFiles), ExistingHSOpts(HSOpts), + ExistingLangOpts(LangOpts), Diags(Diags) {} bool needsImportVisitation() const override { return true; } @@ -77,26 +106,47 @@ class PrebuiltModuleListener : public ASTReaderListener { NewModuleFiles.push_back(Filename.str()); } + bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts, + bool Complain) override { + return checkHeaderSearchPaths( + HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts); + } + private: PrebuiltModuleFilesT &PrebuiltModuleFiles; llvm::SmallVector &NewModuleFiles; + const HeaderSearchOptions &ExistingHSOpts; + const LangOptions &ExistingLangOpts; + DiagnosticsEngine &Diags; }; /// Visit the given prebuilt module and collect all of the modules it /// transitively imports and contributing input files. -static void visitPrebuiltModule(StringRef PrebuiltModuleFilename, +static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, CompilerInstance &CI, - PrebuiltModuleFilesT &ModuleFiles) { + PrebuiltModuleFilesT &ModuleFiles, + DiagnosticsEngine &Diags) { // List of module files to be processed. - llvm::SmallVector Worklist{PrebuiltModuleFilename.str()}; - PrebuiltModuleListener Listener(ModuleFiles, Worklist); - - while (!Worklist.empty()) - ASTReader::readASTFileControlBlock( - Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(), - CI.getPCHContainerReader(), - /*FindModuleFileExtensions=*/false, Listener, - /*ValidateDiagnosticOptions=*/false); + llvm::SmallVector Worklist; + PrebuiltModuleListener Listener( + ModuleFiles, Worklist, CI.getHeaderSearchOpts(), CI.getLangOpts(), Diags); + + if (ASTReader::readASTFileControlBlock( + PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(), + CI.getPCHContainerReader(), + /*FindModuleFileExtensions=*/false, Listener, + /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate)) + return true; + + while (!Worklist.empty()) { + if (ASTReader::readASTFileControlBlock( + Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(), + CI.getPCHContainerReader(), + /*FindModuleFileExtensions=*/false, Listener, + /*ValidateDiagnosticOptions=*/false)) + return true; + } + return false; } /// Transform arbitrary file name into an object-like file name. @@ -183,6 +233,7 @@ class DependencyScanningAction : public tooling::ToolAction { ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false; ScanInstance.getFrontendOpts().ModulesShareFileManager = false; ScanInstance.getHeaderSearchOpts().ModuleFormat = "raw"; + ScanInstance.getHeaderSearchOpts().ModulesIncludeVFSUsage = true; ScanInstance.setFileManager(FileMgr); // Support for virtual file system overlays. @@ -196,9 +247,12 @@ class DependencyScanningAction : public tooling::ToolAction { // will prevent the implicit build to create duplicate modules and will // force reuse of the existing prebuilt module files instead. if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) - visitPrebuiltModule( - ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, - ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles); + if (visitPrebuiltModule( + ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, + ScanInstance, + ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, + ScanInstance.getDiagnostics())) + return false; // Use the dependency scanning optimized file system if requested to do so. if (DepFS) { diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index bfaa897851041..b807dc8432185 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -31,25 +31,55 @@ const std::vector &ModuleDeps::getBuildArguments() { static void optimizeHeaderSearchOpts(HeaderSearchOptions &Opts, ASTReader &Reader, - const serialization::ModuleFile &MF) { - // Only preserve search paths that were used during the dependency scan. - std::vector Entries = Opts.UserEntries; - Opts.UserEntries.clear(); - - llvm::BitVector SearchPathUsage(Entries.size()); - llvm::DenseSet Visited; - std::function VisitMF = - [&](const serialization::ModuleFile *MF) { - SearchPathUsage |= MF->SearchPathUsage; - Visited.insert(MF); - for (const serialization::ModuleFile *Import : MF->Imports) - if (!Visited.contains(Import)) - VisitMF(Import); - }; - VisitMF(&MF); - - for (auto Idx : SearchPathUsage.set_bits()) - Opts.UserEntries.push_back(Entries[Idx]); + const serialization::ModuleFile &MF, + ScanningOptimizations OptimizeArgs) { + if (any(OptimizeArgs & ScanningOptimizations::HeaderSearch)) { + // Only preserve search paths that were used during the dependency scan. + std::vector Entries; + std::swap(Opts.UserEntries, Entries); + + llvm::BitVector SearchPathUsage(Entries.size()); + llvm::DenseSet Visited; + std::function VisitMF = + [&](const serialization::ModuleFile *MF) { + SearchPathUsage |= MF->SearchPathUsage; + Visited.insert(MF); + for (const serialization::ModuleFile *Import : MF->Imports) + if (!Visited.contains(Import)) + VisitMF(Import); + }; + VisitMF(&MF); + + if (SearchPathUsage.size() != Entries.size()) + llvm::report_fatal_error( + "Inconsistent search path options between modules detected"); + + for (auto Idx : SearchPathUsage.set_bits()) + Opts.UserEntries.push_back(std::move(Entries[Idx])); + } + if (any(OptimizeArgs & ScanningOptimizations::VFS)) { + std::vector VFSOverlayFiles; + std::swap(Opts.VFSOverlayFiles, VFSOverlayFiles); + + llvm::BitVector VFSUsage(VFSOverlayFiles.size()); + llvm::DenseSet Visited; + std::function VisitMF = + [&](const serialization::ModuleFile *MF) { + VFSUsage |= MF->VFSUsage; + Visited.insert(MF); + for (const serialization::ModuleFile *Import : MF->Imports) + if (!Visited.contains(Import)) + VisitMF(Import); + }; + VisitMF(&MF); + + if (VFSUsage.size() != VFSOverlayFiles.size()) + llvm::report_fatal_error( + "Inconsistent -ivfsoverlay options between modules detected"); + + for (auto Idx : VFSUsage.set_bits()) + Opts.VFSOverlayFiles.push_back(std::move(VFSOverlayFiles[Idx])); + } } static void optimizeDiagnosticOpts(DiagnosticOptions &Opts, @@ -558,9 +588,11 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { CowCompilerInvocation CI = MDC.getInvocationAdjustedForModuleBuildWithoutOutputs( MD, [&](CowCompilerInvocation &BuildInvocation) { - if (any(MDC.OptimizeArgs & ScanningOptimizations::HeaderSearch)) + if (any(MDC.OptimizeArgs & (ScanningOptimizations::HeaderSearch | + ScanningOptimizations::VFS))) optimizeHeaderSearchOpts(BuildInvocation.getMutHeaderSearchOpts(), - *MDC.ScanInstance.getASTReader(), *MF); + *MDC.ScanInstance.getASTReader(), *MF, + MDC.OptimizeArgs); if (any(MDC.OptimizeArgs & ScanningOptimizations::SystemWarnings)) optimizeDiagnosticOpts( BuildInvocation.getMutDiagnosticOpts(), diff --git a/clang/test/ClangScanDeps/optimize-vfs-edgecases.m b/clang/test/ClangScanDeps/optimize-vfs-edgecases.m new file mode 100644 index 0000000000000..0f18571dd05b2 --- /dev/null +++ b/clang/test/ClangScanDeps/optimize-vfs-edgecases.m @@ -0,0 +1,84 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands.json.in > %t/build/compile-commands.json +// RUN: sed -e "s|DIR|%/t|g" %t/build/vfsoverlay.yaml.in > %t/build/vfsoverlay.yaml +// RUN: sed -e "s|DIR|%/t|g" %t/build/vfs.notyaml.in > %t/build/vfs.notyaml +// RUN: clang-scan-deps -compilation-database %t/build/compile-commands.json \ +// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/deps.db + +// RUN: %deps-to-rsp %t/deps.db --module-name=A > %t/A.rsp +// RUN: cd %t && %clang @%t/A.rsp + +// Check that the following edge cases are handled by ivfsoverlay tracking +// * `-ivfsoverlay` args that depend on earlier `-ivfsoverlay` args. + +//--- build/compile-commands.json.in + +[ +{ + "directory": "DIR", + "command": "clang -c DIR/0.m -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/vfsoverlay.yaml -ivfsoverlay build/vfs.yaml", + "file": "DIR/0.m" +} +] + +//--- build/vfsoverlay.yaml.in + +{ + "version":0, + "case-sensitive":"false", + "roots":[ + { + "contents":[ + { + "external-contents":"DIR/build/vfs.notyaml", + "name":"vfs.yaml", + "type":"file" + } + ], + "name":"DIR/build", + "type":"directory" + } + ] +} + +//--- build/vfs.notyaml.in + +{ + "version":0, + "case-sensitive":"false", + "roots":[ + { + "contents":[ + { + "external-contents":"DIR/build/module.modulemap", + "name":"module.modulemap", + "type":"file" + }, + { + "external-contents":"DIR/build/A.h", + "name":"A.h", + "type":"file" + } + ], + "name":"DIR/modules/A", + "type":"directory" + } + ] +} + +//--- build/module.modulemap + +module A { + umbrella header "A.h" +} + +//--- build/A.h + +typedef int A_t; + +//--- 0.m + +#include + +A_t a = 0; diff --git a/clang/test/ClangScanDeps/optimize-vfs-leak.m b/clang/test/ClangScanDeps/optimize-vfs-leak.m new file mode 100644 index 0000000000000..beda44cdfecb7 --- /dev/null +++ b/clang/test/ClangScanDeps/optimize-vfs-leak.m @@ -0,0 +1,105 @@ +// This test checks that VFS usage doesn't leak between modules. + +// RUN: rm -rf %t && split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/build/cdb.json.in > %t/build/cdb.json +// RUN: sed -e "s|DIR|%/t|g" %t/build/vfs.yaml.in > %t/build/vfs.yaml +// RUN: clang-scan-deps -compilation-database %t/build/cdb.json \ +// RUN: -format experimental-full --optimize-args=vfs > %t/deps.json +// RUN: cat %t/deps.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "B" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "C" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/moduleA/module.modulemap", +// CHECK-NEXT: "command-line": [ +// Module A needs the VFS overlay because its dependency, module B, needs it. +// CHECK: "-ivfsoverlay" +// CHECK-NEXT: "[[PREFIX]]/build/vfs.yaml" +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK: ], +// CHECK-NEXT: "name": "A" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/moduleB/module.modulemap", +// CHECK-NEXT: "command-line": [ +// Module B needs the VFS overlay because it provides the header referred to by the module map. +// CHECK: "-ivfsoverlay" +// CHECK-NEXT: "[[PREFIX]]/build/vfs.yaml" +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK: ], +// CHECK-NEXT: "name": "B" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/moduleC/module.modulemap", +// CHECK-NEXT: "command-line": [ +// Module C doesn't need the VFS overlay. +// CHECK-NOT: "-ivfsoverlay" +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK: ], +// CHECK-NEXT: "name": "C" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK: ] +// CHECK: } + +//--- build/cdb.json.in +[{ + "directory": "DIR", + "command": "clang -c DIR/tu.m -I DIR/moduleA -I DIR/moduleB -I DIR/moduleC -fmodules -fmodules-cache-path=DIR/cache -fimplicit-module-maps -ivfsoverlay DIR/build/vfs.yaml", + "file": "DIR/tu.m" +}] + +//--- build/vfs.yaml.in +{ + "version": 0, + "case-sensitive": "false", + "roots": [ + { + "contents": [ + { + "external-contents": "DIR/build/B.h", + "name": "B.h", + "type": "file" + } + ], + "name": "DIR/moduleB", + "type": "directory" + } + ] +} + +//--- tu.m +@import A; + +//--- moduleA/module.modulemap +module A { header "A.h" } +//--- moduleA/A.h +@import B; +@import C; + +//--- moduleB/module.modulemap +module B { header "B.h" } +//--- build/B.h + +//--- moduleC/module.modulemap +module C { header "C.h" } +//--- moduleC/C.h \ No newline at end of file diff --git a/clang/test/ClangScanDeps/optimize-vfs-pch.m b/clang/test/ClangScanDeps/optimize-vfs-pch.m new file mode 100644 index 0000000000000..e6acb73e1dd34 --- /dev/null +++ b/clang/test/ClangScanDeps/optimize-vfs-pch.m @@ -0,0 +1,129 @@ +// Check that tracking of VFSs works with PCH. + +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands-pch.json.in > %t/build/compile-commands-pch.json +// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands-tu.json.in > %t/build/compile-commands-tu.json +// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands-tu-no-vfs.json.in > %t/build/compile-commands-tu-no-vfs.json +// RUN: sed -e "s|DIR|%/t|g" %t/build/pch-overlay.yaml.in > %t/build/pch-overlay.yaml + +// RUN: clang-scan-deps -compilation-database %t/build/compile-commands-pch.json \ +// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/pch-deps.db +// RUN: %deps-to-rsp %t/pch-deps.db --module-name=A > %t/A.rsp +// RUN: %deps-to-rsp %t/pch-deps.db --module-name=B > %t/B.rsp +// RUN: %deps-to-rsp %t/pch-deps.db --tu-index=0 > %t/pch.rsp +// RUN: %clang @%t/A.rsp +// RUN: %clang @%t/B.rsp +// RUN: %clang @%t/pch.rsp + +// RUN: clang-scan-deps -compilation-database %t/build/compile-commands-tu.json \ +// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/tu-deps.db +// RUN: %deps-to-rsp %t/tu-deps.db --module-name=C > %t/C.rsp +// RUN: %deps-to-rsp %t/tu-deps.db --tu-index=0 > %t/tu.rsp +// RUN: %clang @%t/C.rsp +// RUN: %clang @%t/tu.rsp + +// RUN: not clang-scan-deps -compilation-database %t/build/compile-commands-tu-no-vfs.json \ +// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search 2>&1 | FileCheck %s + +// CHECK: error: PCH was compiled with different VFS overlay files than are currently in use +// CHECK: note: current translation unit has no VFS overlays + +//--- build/compile-commands-pch.json.in + +[ +{ + "directory": "DIR", + "command": "clang -x objective-c-header DIR/pch.h -I DIR/modules/A -I DIR/modules/B -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -o DIR/pch.h.pch -ivfsoverlay DIR/build/pch-overlay.yaml", + "file": "DIR/pch.h" +} +] + +//--- build/compile-commands-tu.json.in + +[ +{ + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -I DIR/modules/A -I DIR/modules/B -I DIR/modules/C -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -include DIR/pch.h -o DIR/tu.o -ivfsoverlay DIR/build/pch-overlay.yaml", + "file": "DIR/tu.m" +} +] + +//--- build/compile-commands-tu-no-vfs.json.in + +[ +{ + "directory": "DIR", + "command": "clang -fsyntax-only DIR/tu.m -I DIR/modules/A -I DIR/modules/B -I DIR/modules/C -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -include DIR/pch.h -o DIR/tu.o", + "file": "DIR/tu.m" +} +] + +//--- build/pch-overlay.yaml.in + +{ + "version":0, + "case-sensitive":"false", + "roots":[ + { + "contents":[ + { + "external-contents":"DIR/build/module.modulemap", + "name":"module.modulemap", + "type":"file" + }, + { + "external-contents":"DIR/build/A.h", + "name":"A.h", + "type":"file" + } + ], + "name":"DIR/modules/A", + "type":"directory" + } + ] +} + +//--- pch.h +#include + +//--- build/module.modulemap + +module A { + umbrella header "A.h" +} + +//--- build/A.h + +typedef int A_t; + +//--- modules/B/module.modulemap + +module B { + umbrella header "B.h" + export * +} + +//--- modules/B/B.h +#include + +typedef int B_t; + +//--- modules/C/module.modulemap + +module C { + umbrella header "C.h" +} + +//--- modules/C/C.h +#include + +typedef int C_t; + +//--- tu.m + +#include + +A_t a = 0; +B_t b = 0; +C_t c = 0; diff --git a/clang/test/ClangScanDeps/optimize-vfs.m b/clang/test/ClangScanDeps/optimize-vfs.m new file mode 100644 index 0000000000000..20c97956087d2 --- /dev/null +++ b/clang/test/ClangScanDeps/optimize-vfs.m @@ -0,0 +1,193 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t +// RUN: sed -e "s|DIR|%/t|g" %t/build/compile-commands.json.in > %t/build/compile-commands.json +// RUN: sed -e "s|DIR|%/t|g" %t/build/vfs.yaml.in > %t/build/vfs.yaml +// RUN: sed -e "s|DIR|%/t|g" %t/build/unused-vfs.yaml.in > %t/build/unused-vfs.yaml +// RUN: sed -e "s|DIR|%/t|g" %t/build/unused-vfs.yaml.in > %t/build/unused2-vfs.yaml +// RUN: clang-scan-deps -compilation-database %t/build/compile-commands.json \ +// RUN: -j 1 -format experimental-full --optimize-args=vfs,header-search > %t/deps.db +// RUN: cat %t/deps.db | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t + +// Check that unused -ivfsoverlay arguments are removed, and that used ones are +// not. + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/A/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NOT: "build/unused-vfs.yaml" +// CHECK: "-ivfsoverlay" +// CHECK-NEXT: "build/vfs.yaml" +// CHECK-NOT: "build/unused-vfs.yaml" +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/build/A.h", +// CHECK-NEXT: "[[PREFIX]]/build/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "A" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/B/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NOT: "-ivfsoverlay" +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules/B/B.h", +// CHECK-NEXT: "[[PREFIX]]/modules/B/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "B" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "B" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/modules/C/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK-NOT: "-ivfsoverlay" +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/modules/B/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/modules/C/C.h", +// CHECK-NEXT: "[[PREFIX]]/modules/C/module.modulemap" +// CHECK-NEXT: ], +// CHECK-NEXT: "name": "C" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [ +// CHECK: ] +// CHECK: } + +//--- build/compile-commands.json.in + +[ +{ + "directory": "DIR", + "command": "clang -c DIR/0.m -Imodules/A -Imodules/B -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/unused-vfs.yaml -ivfsoverlay build/unused2-vfs.yaml -ivfsoverlay build/vfs.yaml", + "file": "DIR/0.m" +}, +{ + "directory": "DIR", + "command": "clang -c DIR/A.m -Imodules/A -Imodules/B -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/vfs.yaml -ivfsoverlay build/unused-vfs.yaml", + "file": "DIR/A.m" +}, +{ + "directory": "DIR", + "command": "clang -c DIR/B.m -Imodules/B -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/unused-vfs.yaml -ivfsoverlay build/vfs.yaml", + "file": "DIR/B.m" +}, +{ + "directory": "DIR", + "command": "clang -c DIR/C.m -Imodules/A -Imodules/B -Imodules/C -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-module-maps -ivfsoverlay build/vfs.yaml -ivfsoverlay build/unused-vfs.yaml", + "file": "DIR/C.m" +} +] + +//--- build/vfs.yaml.in + +{ + "version":0, + "case-sensitive":"false", + "roots":[ + { + "contents":[ + { + "external-contents":"DIR/build/module.modulemap", + "name":"module.modulemap", + "type":"file" + }, + { + "external-contents":"DIR/build/A.h", + "name":"A.h", + "type":"file" + } + ], + "name":"DIR/modules/A", + "type":"directory" + } + ] +} + +//--- build/unused-vfs.yaml.in + +{ + "version":0, + "case-sensitive":"false", + "roots":[ + { + "contents":[ + { + "external-contents":"DIR/build/module.modulemap", + "name":"module.modulemap", + "type":"file" + } + ], + "name":"DIR/modules/D", + "type":"directory" + } + ] +} + +//--- build/module.modulemap + +module A { + umbrella header "A.h" +} + +//--- build/A.h + +typedef int A_t; + +//--- modules/B/module.modulemap + +module B { + umbrella header "B.h" +} + +//--- modules/B/B.h + +typedef int B_t; + +//--- modules/C/module.modulemap + +module C { + umbrella header "C.h" +} + +//--- modules/C/C.h + +@import B; + +typedef B_t C_t; + +//--- 0.m + +#include + +A_t a = 0; + +//--- A.m + +#include + +A_t a = 0; + +//--- B.m + +#include + +B_t b = 0; + +//--- C.m + +#include + +C_t b = 0; diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index fb42dc1085afb..76337664c45e3 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -156,6 +156,7 @@ static void ParseArgs(int argc, char **argv) { .Case("none", ScanningOptimizations::None) .Case("header-search", ScanningOptimizations::HeaderSearch) .Case("system-warnings", ScanningOptimizations::SystemWarnings) + .Case("vfs", ScanningOptimizations::VFS) .Case("all", ScanningOptimizations::All) .Default(std::nullopt); if (!Optimization) { diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h index 1c3d8e985592b..1a5ea677db74d 100644 --- a/llvm/include/llvm/Support/VirtualFileSystem.h +++ b/llvm/include/llvm/Support/VirtualFileSystem.h @@ -15,12 +15,14 @@ #define LLVM_SUPPORT_VIRTUALFILESYSTEM_H #include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/STLFunctionalExtras.h" #include "llvm/Support/Chrono.h" -#include "llvm/Support/ErrorOr.h" #include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/ExtensibleRTTI.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" #include "llvm/Support/SourceMgr.h" @@ -261,8 +263,10 @@ class recursive_directory_iterator { }; /// The virtual file system interface. -class FileSystem : public llvm::ThreadSafeRefCountedBase { +class FileSystem : public llvm::ThreadSafeRefCountedBase, + public RTTIExtends { public: + static const char ID; virtual ~FileSystem(); /// Get the status of the entry at \p Path, if one exists. @@ -321,6 +325,13 @@ class FileSystem : public llvm::ThreadSafeRefCountedBase { printImpl(OS, Type, IndentLevel); } + using VisitCallbackTy = llvm::function_ref; + virtual void visitChildFileSystems(VisitCallbackTy Callback) {} + void visit(VisitCallbackTy Callback) { + Callback(*this); + visitChildFileSystems(Callback); + } + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void dump() const; #endif @@ -360,7 +371,7 @@ std::unique_ptr createPhysicalFileSystem(); /// top-most (most recently added) directory are used. When there is a file /// that exists in more than one file system, the file in the top-most file /// system overrides the other(s). -class OverlayFileSystem : public FileSystem { +class OverlayFileSystem : public RTTIExtends { using FileSystemList = SmallVector, 1>; /// The stack of file systems, implemented as a list in order of @@ -368,6 +379,7 @@ class OverlayFileSystem : public FileSystem { FileSystemList FSList; public: + static const char ID; OverlayFileSystem(IntrusiveRefCntPtr Base); /// Pushes a file system on top of the stack. @@ -412,13 +424,15 @@ class OverlayFileSystem : public FileSystem { protected: void printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const override; + void visitChildFileSystems(VisitCallbackTy Callback) override; }; /// By default, this delegates all calls to the underlying file system. This /// is useful when derived file systems want to override some calls and still /// proxy other calls. -class ProxyFileSystem : public FileSystem { +class ProxyFileSystem : public RTTIExtends { public: + static const char ID; explicit ProxyFileSystem(IntrusiveRefCntPtr FS) : FS(std::move(FS)) {} @@ -448,11 +462,17 @@ class ProxyFileSystem : public FileSystem { protected: FileSystem &getUnderlyingFS() const { return *FS; } + void visitChildFileSystems(VisitCallbackTy Callback) override { + if (FS) { + Callback(*FS); + FS->visitChildFileSystems(Callback); + } + } private: IntrusiveRefCntPtr FS; - virtual void anchor(); + virtual void anchor() override; }; namespace detail { @@ -495,11 +515,15 @@ class NamedNodeOrError { } // namespace detail /// An in-memory file system. -class InMemoryFileSystem : public FileSystem { +class InMemoryFileSystem : public RTTIExtends { std::unique_ptr Root; std::string WorkingDirectory; bool UseNormalizedPaths = true; +public: + static const char ID; + +private: using MakeNodeFn = llvm::function_ref( detail::NewInMemoryNodeInfo)>; @@ -736,8 +760,10 @@ class RedirectingFileSystemParser; /// FIXME: 'use-external-name' causes behaviour that's inconsistent with how /// "real" filesystems behave. Maybe there should be a separate channel for /// this information. -class RedirectingFileSystem : public vfs::FileSystem { +class RedirectingFileSystem + : public RTTIExtends { public: + static const char ID; enum EntryKind { EK_Directory, EK_DirectoryRemap, EK_File }; enum NameKind { NK_NotSet, NK_External, NK_Virtual }; @@ -973,6 +999,13 @@ class RedirectingFileSystem : public vfs::FileSystem { /// names of files. This global value is overridable on a per-file basis. bool UseExternalNames = true; + /// True if this FS has redirected a lookup. This does not include + /// fallthrough. + mutable bool HasBeenUsed = false; + + /// Used to enable or disable updating `HasBeenUsed`. + bool UsageTrackingActive = false; + /// Determines the lookups to perform, as well as their order. See /// \c RedirectKind for details. RedirectKind Redirection = RedirectKind::Fallthrough; @@ -1043,11 +1076,17 @@ class RedirectingFileSystem : public vfs::FileSystem { std::vector getRoots() const; + bool hasBeenUsed() const { return HasBeenUsed; }; + void clearHasBeenUsed() { HasBeenUsed = false; } + + void setUsageTrackingActive(bool Active) { UsageTrackingActive = Active; } + void printEntry(raw_ostream &OS, Entry *E, unsigned IndentLevel = 0) const; protected: void printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const override; + void visitChildFileSystems(VisitCallbackTy Callback) override; }; /// Collect all pairs of entries from the diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp index d6c787cb91b34..c341170688037 100644 --- a/llvm/lib/Support/VirtualFileSystem.cpp +++ b/llvm/lib/Support/VirtualFileSystem.cpp @@ -480,6 +480,13 @@ OverlayFileSystem::getRealPath(const Twine &Path, return errc::no_such_file_or_directory; } +void OverlayFileSystem::visitChildFileSystems(VisitCallbackTy Callback) { + for (IntrusiveRefCntPtr FS : overlays_range()) { + Callback(*FS); + FS->visitChildFileSystems(Callback); + } +} + void OverlayFileSystem::printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const { printIndent(OS, IndentLevel); @@ -1581,6 +1588,13 @@ void RedirectingFileSystem::printEntry(raw_ostream &OS, } } +void RedirectingFileSystem::visitChildFileSystems(VisitCallbackTy Callback) { + if (ExternalFS) { + Callback(*ExternalFS); + ExternalFS->visitChildFileSystems(Callback); + } +} + /// A helper class to hold the common YAML parsing state. class llvm::vfs::RedirectingFileSystemParser { yaml::Stream &Stream; @@ -2263,12 +2277,18 @@ RedirectingFileSystem::makeCanonical(SmallVectorImpl &Path) const { ErrorOr RedirectingFileSystem::lookupPath(StringRef Path) const { + // RedirectOnly means the VFS is always used. + if (UsageTrackingActive && Redirection == RedirectKind::RedirectOnly) + HasBeenUsed = true; + sys::path::const_iterator Start = sys::path::begin(Path); sys::path::const_iterator End = sys::path::end(Path); llvm::SmallVector Entries; for (const auto &Root : Roots) { ErrorOr Result = lookupPathImpl(Start, End, Root.get(), Entries); + if (UsageTrackingActive && Result && isa(Result->E)) + HasBeenUsed = true; if (Result || Result.getError() != llvm::errc::no_such_file_or_directory) { Result->Parents = std::move(Entries); return Result; @@ -2863,3 +2883,9 @@ recursive_directory_iterator::increment(std::error_code &EC) { return *this; } + +const char FileSystem::ID = 0; +const char OverlayFileSystem::ID = 0; +const char ProxyFileSystem::ID = 0; +const char InMemoryFileSystem::ID = 0; +const char RedirectingFileSystem::ID = 0; diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp index 54a6e0fbc0760..d4abbb4345873 100644 --- a/llvm/unittests/Support/VirtualFileSystemTest.cpp +++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp @@ -895,6 +895,47 @@ TEST(VirtualFileSystemTest, HiddenInIteration) { } } +TEST(VirtualFileSystemTest, Visit) { + IntrusiveRefCntPtr Base(new DummyFileSystem()); + IntrusiveRefCntPtr Middle(new DummyFileSystem()); + IntrusiveRefCntPtr Top(new DummyFileSystem()); + IntrusiveRefCntPtr O( + new vfs::OverlayFileSystem(Base)); + O->pushOverlay(Middle); + O->pushOverlay(Top); + + auto YAML = + MemoryBuffer::getMemBuffer("{\n" + " 'version': 0,\n" + " 'redirecting-with': 'redirect-only',\n" + " 'roots': [\n" + " {\n" + " 'type': 'file',\n" + " 'name': '/vfile',\n" + " 'external-contents': '/a',\n" + " }," + " ]\n" + "}"); + + IntrusiveRefCntPtr Redirecting = + vfs::RedirectingFileSystem::create(std::move(YAML), nullptr, "", nullptr, + O) + .release(); + + vfs::ProxyFileSystem PFS(Redirecting); + + std::vector FSs; + PFS.visit([&](const vfs::FileSystem &FS) { FSs.push_back(&FS); }); + + ASSERT_EQ(size_t(6), FSs.size()); + EXPECT_TRUE(isa(FSs[0])); + EXPECT_TRUE(isa(FSs[1])); + EXPECT_TRUE(isa(FSs[2])); + EXPECT_TRUE(isa(FSs[3])); + EXPECT_TRUE(isa(FSs[4])); + EXPECT_TRUE(isa(FSs[5])); +} + TEST(OverlayFileSystemTest, PrintOutput) { auto Dummy = makeIntrusiveRefCnt(); auto Overlay1 = makeIntrusiveRefCnt(Dummy); @@ -3244,3 +3285,48 @@ TEST(RedirectingFileSystemTest, PrintOutput) { " DummyFileSystem (RecursiveContents)\n", Output); } + +TEST(RedirectingFileSystemTest, Used) { + auto Dummy = makeIntrusiveRefCnt(); + auto YAML1 = + MemoryBuffer::getMemBuffer("{\n" + " 'version': 0,\n" + " 'redirecting-with': 'fallthrough',\n" + " 'roots': [\n" + " {\n" + " 'type': 'file',\n" + " 'name': '/vfile1',\n" + " 'external-contents': '/a',\n" + " }," + " ]\n" + "}"); + auto YAML2 = + MemoryBuffer::getMemBuffer("{\n" + " 'version': 0,\n" + " 'redirecting-with': 'fallthrough',\n" + " 'roots': [\n" + " {\n" + " 'type': 'file',\n" + " 'name': '/vfile2',\n" + " 'external-contents': '/b',\n" + " }," + " ]\n" + "}"); + + Dummy->addRegularFile("/a"); + Dummy->addRegularFile("/b"); + + IntrusiveRefCntPtr Redirecting1 = + vfs::RedirectingFileSystem::create(std::move(YAML1), nullptr, "", nullptr, + Dummy) + .release(); + auto Redirecting2 = vfs::RedirectingFileSystem::create( + std::move(YAML2), nullptr, "", nullptr, Redirecting1); + + Redirecting1->setUsageTrackingActive(true); + Redirecting2->setUsageTrackingActive(true); + EXPECT_TRUE(Redirecting2->exists("/vfile1")); + EXPECT_TRUE(Redirecting2->exists("/b")); + EXPECT_TRUE(Redirecting1->hasBeenUsed()); + EXPECT_FALSE(Redirecting2->hasBeenUsed()); +}