diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h index 62d7e16e1251c..b27d2c0ce9615 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h @@ -38,9 +38,8 @@ class DependencyScanningCASFilesystem public: static const char ID; - DependencyScanningCASFilesystem( - IntrusiveRefCntPtr WorkerFS, - llvm::cas::ActionCache &Cache); + DependencyScanningCASFilesystem(DependencyScanningService &Service, + IntrusiveRefCntPtr WorkerFS); ~DependencyScanningCASFilesystem(); @@ -109,6 +108,8 @@ class DependencyScanningCASFilesystem llvm::cas::ObjectStore &CAS; llvm::cas::ActionCache &Cache; + /// The service associated with this VFS. + DependencyScanningService &Service; std::optional ClangFullVersionID; std::optional DepDirectivesID; std::optional EmptyBlobID; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h index c452398c1b637..dc161401f0add 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h @@ -25,6 +25,8 @@ namespace clang { namespace tooling { namespace dependencies { +class DependencyScanningService; + using DependencyDirectivesTy = SmallVector; @@ -387,7 +389,7 @@ class DependencyScanningWorkerFilesystem static const char ID; DependencyScanningWorkerFilesystem( - DependencyScanningFilesystemSharedCache &SharedCache, + DependencyScanningService &Service, IntrusiveRefCntPtr FS); llvm::ErrorOr status(const Twine &Path) override; @@ -476,10 +478,7 @@ class DependencyScanningWorkerFilesystem /// Returns entry associated with the unique ID in the shared cache or nullptr /// if none is found. const CachedFileSystemEntry * - findSharedEntryByUID(llvm::vfs::Status Stat) const { - return SharedCache.getShardForUID(Stat.getUniqueID()) - .findEntryByUID(Stat.getUniqueID()); - } + findSharedEntryByUID(llvm::vfs::Status Stat) const; /// Associates the given entry with the filename in the local cache and /// returns it. @@ -493,20 +492,14 @@ class DependencyScanningWorkerFilesystem /// some. Otherwise, constructs new one with the given error code, associates /// it with the filename and returns the result. const CachedFileSystemEntry & - getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) { - return SharedCache.getShardForFilename(Filename) - .getOrEmplaceEntryForFilename(Filename, EC); - } + getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC); /// Returns entry associated with the filename in the shared cache if there is /// some. Otherwise, associates the given entry with the filename and returns /// it. const CachedFileSystemEntry & getOrInsertSharedEntryForFilename(StringRef Filename, - const CachedFileSystemEntry &Entry) { - return SharedCache.getShardForFilename(Filename) - .getOrInsertEntryForFilename(Filename, Entry); - } + const CachedFileSystemEntry &Entry); void printImpl(raw_ostream &OS, PrintType Type, unsigned IndentLevel) const override { @@ -519,8 +512,9 @@ class DependencyScanningWorkerFilesystem /// VFS. bool shouldBypass(StringRef Path) const; - /// The global cache shared between worker threads. - DependencyScanningFilesystemSharedCache &SharedCache; + /// The service associated with this VFS. + DependencyScanningService &Service; + /// The local cache is used by the worker thread to cache file system queries /// locally instead of querying the global cache every time. DependencyScanningFilesystemLocalCache LocalCache; diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h index 5c86186a466df..aa8136db62e4f 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -97,6 +97,8 @@ enum class ScanningOptimizations { #undef DSS_LAST_BITMASK_ENUM +bool shouldNegativeStatCacheDefault(); + /// The dependency scanning service contains shared configuration and state that /// is used by the individual dependency scanning workers. class DependencyScanningService { @@ -109,7 +111,8 @@ class DependencyScanningService { ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default, bool EagerLoadModules = false, bool TraceVFS = false, std::time_t BuildSessionTimestamp = - llvm::sys::toTimeT(std::chrono::system_clock::now())); + llvm::sys::toTimeT(std::chrono::system_clock::now()), + bool CacheNegativeStats = shouldNegativeStatCacheDefault()); ScanningMode getMode() const { return Mode; } @@ -121,6 +124,8 @@ class DependencyScanningService { bool shouldTraceVFS() const { return TraceVFS; } + bool shouldCacheNegativeStats() const { return CacheNegativeStats; } + DependencyScanningFilesystemSharedCache &getSharedCache() { assert(!SharedFS && "Expected not to have a CASFS"); assert(SharedCache && "Expected a shared cache"); @@ -152,6 +157,7 @@ class DependencyScanningService { const bool EagerLoadModules; /// Whether to trace VFS accesses. const bool TraceVFS; + const bool CacheNegativeStats; /// Shared CachingOnDiskFileSystem. Set to nullptr to not use CAS dependency /// scanning. IntrusiveRefCntPtr SharedFS; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp index a32afcc55c715..fb1f5be90ba92 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningCASFilesystem.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/Basic/Version.h" #include "clang/Lex/DependencyDirectivesScanner.h" #include "llvm/CAS/ActionCache.h" @@ -35,10 +36,10 @@ static void reportAsFatalIfError(llvm::Error E) { using llvm::Error; DependencyScanningCASFilesystem::DependencyScanningCASFilesystem( - IntrusiveRefCntPtr WorkerFS, - llvm::cas::ActionCache &Cache) - : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), Cache(Cache) { -} + DependencyScanningService &Service, + IntrusiveRefCntPtr WorkerFS) + : FS(WorkerFS), Entries(EntryAlloc), CAS(WorkerFS->getCAS()), + Cache(*Service.getCache()), Service(Service) {} const char DependencyScanningCASFilesystem::ID = 0; DependencyScanningCASFilesystem::~DependencyScanningCASFilesystem() = default; @@ -233,7 +234,7 @@ DependencyScanningCASFilesystem::lookupPath(const Twine &Path) { llvm::ErrorOr MaybeStatus = getCachingFS().statusAndFileID(PathRef, FileID); if (!MaybeStatus) { - if (shouldCacheStatFailures(PathRef)) + if (Service.shouldCacheNegativeStats() && shouldCacheStatFailures(PathRef)) Entries[PathRef].EC = MaybeStatus.getError(); return LookupPathResult{nullptr, MaybeStatus.getError()}; } diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp index f606b9c3e990e..a473afbf3a1fd 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "llvm/CAS/CASFileSystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Threading.h" @@ -255,19 +256,19 @@ bool DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const { } DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem( - DependencyScanningFilesystemSharedCache &SharedCache, + DependencyScanningService &Service, IntrusiveRefCntPtr FS) : llvm::RTTIExtends(std::move(FS)), - SharedCache(SharedCache), - WorkingDirForCacheLookup(llvm::errc::invalid_argument) { + Service(Service), WorkingDirForCacheLookup(llvm::errc::invalid_argument) { updateWorkingDirForCacheLookup(); } const CachedFileSystemEntry & DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID( TentativeEntry TEntry) { - auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID()); + auto &Shard = + Service.getSharedCache().getShardForUID(TEntry.Status.getUniqueID()); return Shard.getOrEmplaceEntryForUID( TEntry.Status.getUniqueID(), std::move(TEntry.Status), std::move(TEntry.Contents), std::move(TEntry.CASContents)); @@ -278,18 +279,50 @@ DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough( StringRef Filename) { if (const auto *Entry = LocalCache.findEntryByFilename(Filename)) return Entry; - auto &Shard = SharedCache.getShardForFilename(Filename); + auto &Shard = Service.getSharedCache().getShardForFilename(Filename); if (const auto *Entry = Shard.findEntryByFilename(Filename)) return &LocalCache.insertEntryForFilename(Filename, *Entry); return nullptr; } +const CachedFileSystemEntry * +DependencyScanningWorkerFilesystem::findSharedEntryByUID( + llvm::vfs::Status Stat) const { + return Service.getSharedCache() + .getShardForUID(Stat.getUniqueID()) + .findEntryByUID(Stat.getUniqueID()); +} + +const CachedFileSystemEntry & +DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForFilename( + StringRef Filename, std::error_code EC) { + return Service.getSharedCache() + .getShardForFilename(Filename) + .getOrEmplaceEntryForFilename(Filename, EC); +} + +const CachedFileSystemEntry & +DependencyScanningWorkerFilesystem::getOrInsertSharedEntryForFilename( + StringRef Filename, const CachedFileSystemEntry &Entry) { + return Service.getSharedCache() + .getShardForFilename(Filename) + .getOrInsertEntryForFilename(Filename, Entry); +} + llvm::ErrorOr DependencyScanningWorkerFilesystem::computeAndStoreResult( StringRef OriginalFilename, StringRef FilenameForLookup) { llvm::ErrorOr Stat = getUnderlyingFS().status(OriginalFilename); if (!Stat) { + // rdar://148027982 + // rdar://127079541 + // Negative caching directories can cause build failures due to incorrectly + // configured projects. + StringRef Ext = llvm::sys::path::extension(OriginalFilename); + if (!Service.shouldCacheNegativeStats() || Ext.empty() || Ext == ".framework") + return Stat.getError(); + const auto &Entry = getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError()); return insertLocalEntryForFilename(FilenameForLookup, Entry); @@ -478,7 +511,8 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path, return HandleCachedRealPath(*RealPath); // If we have the result in the shared cache, cache it locally. - auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup); + auto &Shard = + Service.getSharedCache().getShardForFilename(*FilenameForLookup); if (const auto *ShardRealPath = Shard.findRealPathByFilename(*FilenameForLookup)) { const auto &RealPath = LocalCache.insertRealPathForFilename( diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp index 846b0dd6460d3..cf6e73b847aae 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp @@ -11,21 +11,36 @@ #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" +#include "llvm/Support/Process.h" using namespace clang; using namespace tooling; using namespace dependencies; +bool clang::tooling::dependencies::shouldNegativeStatCacheDefault() { + if (std::optional MaybeNegStats = + llvm::sys::Process::GetEnv("CLANG_SCAN_CACHE_NEGATIVE_STATS")) { + if (MaybeNegStats->empty()) + return true; + return llvm::StringSwitch(*MaybeNegStats) + .Case("1", true) + .Case("0", false) + .Default(false); + } + return false; +} + DependencyScanningService::DependencyScanningService( ScanningMode Mode, ScanningOutputFormat Format, CASOptions CASOpts, std::shared_ptr CAS, std::shared_ptr Cache, IntrusiveRefCntPtr SharedFS, ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS, - std::time_t BuildSessionTimestamp) + std::time_t BuildSessionTimestamp, bool CacheNegativeStats) : Mode(Mode), Format(Format), CASOpts(std::move(CASOpts)), CAS(std::move(CAS)), Cache(std::move(Cache)), OptimizeArgs(OptimizeArgs), EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS), + CacheNegativeStats(CacheNegativeStats), SharedFS(std::move(SharedFS)), BuildSessionTimestamp(BuildSessionTimestamp) { if (!this->SharedFS) diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index d187b2af2e3d5..cd0bf15a7af8a 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -34,7 +34,7 @@ DependencyScanningWorker::DependencyScanningWorker( if (Service.useCASFS()) { CacheFS = Service.getSharedFS().createProxyFS(); - DepCASFS = new DependencyScanningCASFilesystem(CacheFS, *Service.getCache()); + DepCASFS = new DependencyScanningCASFilesystem(Service, CacheFS); BaseFS = DepCASFS; return; } @@ -42,7 +42,7 @@ DependencyScanningWorker::DependencyScanningWorker( switch (Service.getMode()) { case ScanningMode::DependencyDirectivesScan: DepFS = llvm::makeIntrusiveRefCnt( - Service.getSharedCache(), FS); + Service, FS); BaseFS = DepFS; break; case ScanningMode::CanonicalPreprocessing: diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index b43df0329125f..3d310be30e4a7 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -92,6 +92,7 @@ static ScanningOutputFormat Format = ScanningOutputFormat::Make; static ScanningOptimizations OptimizeArgs; static std::string ModuleFilesDir; static bool EagerLoadModules; +static bool CacheNegativeStats = true; static unsigned NumThreads = 0; static std::string CompilationDB; static std::optional ModuleName; @@ -212,6 +213,8 @@ static void ParseArgs(int argc, char **argv) { EagerLoadModules = Args.hasArg(OPT_eager_load_pcm); + CacheNegativeStats = !Args.hasArg(OPT_no_cache_negative_stats); + if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) { StringRef S{A->getValue()}; if (!llvm::to_integer(S, NumThreads, 0)) { @@ -1496,9 +1499,10 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { }); }; - DependencyScanningService Service(ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, - EagerLoadModules, - /*TraceVFS=*/Verbose); + DependencyScanningService Service( + ScanMode, Format, CASOpts, CAS, Cache, FS, OptimizeArgs, EagerLoadModules, + /*TraceVFS=*/Verbose, + llvm::sys::toTimeT(std::chrono::system_clock::now()), CacheNegativeStats); llvm::Timer T; T.startTimer(); diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td index efcf950780f73..6d73535d31874 100644 --- a/clang/tools/clang-scan-deps/Opts.td +++ b/clang/tools/clang-scan-deps/Opts.td @@ -22,6 +22,7 @@ defm module_files_dir : Eq<"module-files-dir", def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">, HelpText<"Which command-line arguments of modules to optimize">; def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of lazily on import)">; +def no_cache_negative_stats : F<"no-cache-negative-stats", "Don't cache stat failures">; def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">; diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp index 5e45982050641..637bc8138536a 100644 --- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp @@ -235,9 +235,12 @@ TEST(DependencyScanner, ScanDepsWithFS) { VFS->addFile(TestPath, 0, llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"\n")); - DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, - ScanningOutputFormat::Make, CASOptions(), - nullptr, nullptr, nullptr); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + CASOptions(), nullptr, nullptr, nullptr, ScanningOptimizations::Default, + /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); DependencyScanningTool ScanTool(Service, VFS); std::string DepFile; @@ -259,12 +262,14 @@ TEST(DependencyScanner, DepScanFSWithCASProvider) { std::unique_ptr CASFS = llvm::cas::createCASProvidingFileSystem(DB, FS); - DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, - ScanningOutputFormat::Make, CASOptions(), - nullptr, nullptr, nullptr); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + CASOptions(), nullptr, nullptr, nullptr, ScanningOptimizations::Default, + /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); { - DependencyScanningWorkerFilesystem DepFS(Service.getSharedCache(), - std::move(CASFS)); + DependencyScanningWorkerFilesystem DepFS(Service, std::move(CASFS)); llvm::ErrorOr> File = DepFS.openFileForRead(Path); ASSERT_TRUE(File); @@ -286,7 +291,7 @@ TEST(DependencyScanner, DepScanFSWithCASProvider) { // Check that even though we pass a new InMemoryFileSystem instance here the // DependencyScanningService's SharedCache cached the file's buffer and // cas::ObjectRef and will be able to provide it. - DependencyScanningWorkerFilesystem DepFS(Service.getSharedCache(), + DependencyScanningWorkerFilesystem DepFS(Service, new llvm::vfs::InMemoryFileSystem); DepFS.setCurrentWorkingDirectory("/root"); llvm::ErrorOr> File = @@ -348,9 +353,12 @@ TEST(DependencyScanner, ScanDepsWithModuleLookup) { auto InterceptFS = llvm::makeIntrusiveRefCnt(VFS); - DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, - ScanningOutputFormat::Make, CASOptions(), - nullptr, nullptr, nullptr); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + CASOptions(), nullptr, nullptr, nullptr, ScanningOptimizations::Default, + /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); DependencyScanningTool ScanTool(Service, InterceptFS); // This will fail with "fatal error: module 'Foo' not found" but it doesn't @@ -381,9 +389,12 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) { llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"\n")); VFS->addFile(AsmPath, 0, llvm::MemoryBuffer::getMemBuffer("")); - DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, - ScanningOutputFormat::Make, CASOptions(), - nullptr, nullptr, nullptr); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + CASOptions(), nullptr, nullptr, nullptr, ScanningOptimizations::Default, + /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); DependencyScanningWorker Worker(Service, VFS); llvm::DenseSet AlreadySeen; @@ -446,3 +457,115 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) { EXPECT_TRUE(DiagConsumer.Finished); } } + +TEST(DependencyScanner, NoNegativeCache) { + StringRef CWD = "/root"; + + auto VFS = new llvm::vfs::InMemoryFileSystem(); + VFS->setCurrentWorkingDirectory(CWD); + auto Sept = llvm::sys::path::get_separator(); + std::string HeaderPath = + std::string(llvm::formatv("{0}root{0}header.h", Sept)); + std::string Test0Path = + std::string(llvm::formatv("{0}root{0}test0.cpp", Sept)); + std::string Test1Path = + std::string(llvm::formatv("{0}root{0}test1.cpp", Sept)); + + VFS->addFile(Test0Path, 0, + llvm::MemoryBuffer::getMemBuffer( + "#if __has_include(\"header.h\")\n#endif")); + VFS->addFile(Test1Path, 0, + llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"")); + + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + CASOptions(), nullptr, nullptr, nullptr, ScanningOptimizations::All, + false, false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + false); + DependencyScanningTool ScanTool(Service, VFS); + + std::vector CommandLine0 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test0.cpp", + "-o" + "test0.cpp.o"}; + std::vector CommandLine1 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test1.cpp", + "-o" + "test1.cpp.o"}; + + std::string Result; + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result), + llvm::Succeeded()); + + VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result), + llvm::Succeeded()); +} + +TEST(DependencyScanner, NoNegativeCacheCAS) { + StringRef CWD = "/root"; + + std::shared_ptr DB = llvm::cas::createInMemoryCAS(); + std::shared_ptr Cache = llvm::cas::createInMemoryActionCache(); + auto VFS = llvm::makeIntrusiveRefCnt(); + VFS->setCurrentWorkingDirectory(CWD); + std::unique_ptr CASFS = + llvm::cas::createCASProvidingFileSystem(DB, VFS); + + auto Sept = llvm::sys::path::get_separator(); + std::string HeaderPath = + std::string(llvm::formatv("{0}root{0}header.h", Sept)); + std::string Test0Path = + std::string(llvm::formatv("{0}root{0}test0.cpp", Sept)); + std::string Test1Path = + std::string(llvm::formatv("{0}root{0}test1.cpp", Sept)); + + VFS->addFile(Test0Path, 0, + llvm::MemoryBuffer::getMemBuffer( + "#if __has_include(\"header.h\")\n#endif")); + VFS->addFile(Test1Path, 0, + llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"")); + + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::FullIncludeTree, + CASOptions(), DB, Cache, nullptr, ScanningOptimizations::Default, + /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + /*CacheNegativeStats=*/false); + DependencyScanningTool ScanTool(Service, VFS); + + std::vector CommandLine0 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test0.cpp", + "-o" + "test0.cpp.o"}; + std::vector CommandLine1 = {"clang", + "-target", + "x86_64-apple-macosx10.7", + "-c", + "test1.cpp", + "-o" + "test1.cpp.o"}; + + std::string Result; + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result), + llvm::Succeeded()); + + VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + + ASSERT_THAT_ERROR( + ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result), + llvm::Succeeded()); +} diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp index 7988e31c76cf8..9f02390ec67cb 100644 --- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp +++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp @@ -7,9 +7,11 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "clang/CAS/CASOptions.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/ADT/SmallString.h" +#include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/Support/VirtualFileSystem.h" #include "gtest/gtest.h" @@ -21,9 +23,14 @@ TEST(DependencyScanningWorkerFilesystem, CacheStatusFailures) { auto InstrumentingFS = llvm::makeIntrusiveRefCnt(InMemoryFS); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); - DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); + DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS); DepFS.status("/foo.c"); EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u); @@ -47,9 +54,14 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) { auto InstrumentingFS = llvm::makeIntrusiveRefCnt(InMemoryFS); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); - DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); + DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS); { llvm::SmallString<128> Result; @@ -82,8 +94,13 @@ TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) { InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer("")); InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer("")); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS); // Success. { @@ -135,8 +152,13 @@ TEST(DependencyScanningFilesystem, CacheStatOnExists) { InMemoryFS->setCurrentWorkingDirectory("/"); InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer("")); InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer("")); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); DepFS.status("/foo"); DepFS.status("/foo"); @@ -158,8 +180,13 @@ TEST(DependencyScanningFilesystem, CacheStatFailures) { auto InstrumentingFS = llvm::makeIntrusiveRefCnt(InMemoryFS); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS); DepFS.status("/dir"); DepFS.status("/dir"); @@ -185,8 +212,13 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) { auto InMemoryFS = llvm::makeIntrusiveRefCnt(); InMemoryFS->setCurrentWorkingDirectory("/"); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS); bool Path1Exists = DepFS.exists("/path1.suffix"); ASSERT_EQ(Path1Exists, false); @@ -198,7 +230,8 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) { // DepFS's eyes. ASSERT_EQ(Path1Exists, false); - auto InvalidEntries = SharedCache.getOutOfDateEntries(*InMemoryFS); + auto InvalidEntries = + Service.getSharedCache().getOutOfDateEntries(*InMemoryFS); EXPECT_EQ(InvalidEntries.size(), 1u); ASSERT_STREQ("/path1.suffix", InvalidEntries[0].Path); @@ -210,8 +243,13 @@ TEST(DependencyScanningFilesystem, DiagnoseCachedFileSizeChange) { InMemoryFS1->setCurrentWorkingDirectory("/"); InMemoryFS2->setCurrentWorkingDirectory("/"); - DependencyScanningFilesystemSharedCache SharedCache; - DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS1); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), nullptr, nullptr, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS1); InMemoryFS1->addFile("/path1.suffix", 0, llvm::MemoryBuffer::getMemBuffer("")); @@ -224,7 +262,8 @@ TEST(DependencyScanningFilesystem, DiagnoseCachedFileSizeChange) { // Check against the new file system. InMemoryFS2 could be the underlying // physical system in the real world. - auto InvalidEntries = SharedCache.getOutOfDateEntries(*InMemoryFS2); + auto InvalidEntries = + Service.getSharedCache().getOutOfDateEntries(*InMemoryFS2); ASSERT_EQ(InvalidEntries.size(), 1u); ASSERT_STREQ("/path1.suffix", InvalidEntries[0].Path); diff --git a/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp index de2e308407020..1095f5e4863f5 100644 --- a/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp +++ b/clang/unittests/Tooling/DependencyScanningCASFilesystemTest.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "clang/Tooling/DependencyScanning/DependencyScanningCASFilesystem.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CachingOnDiskFileSystem.h" #include "llvm/CAS/ObjectStore.h" @@ -32,10 +33,16 @@ TEST(DependencyScanningCASFilesystem, FilenameSpelling) { TempLink TestLink(TestDir.path("File.h"), TestDir.path("SymFile.h")); #endif - std::unique_ptr CAS = llvm::cas::createInMemoryCAS(); - std::unique_ptr Cache = llvm::cas::createInMemoryActionCache(); + std::shared_ptr CAS = llvm::cas::createInMemoryCAS(); + std::shared_ptr Cache = llvm::cas::createInMemoryActionCache(); auto CacheFS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); - DependencyScanningCASFilesystem FS(CacheFS, *Cache); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), CAS, Cache, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningCASFilesystem FS(Service, CacheFS); EXPECT_EQ(FS.status(TestFile.path()).getError(), std::error_code()); auto Directives = FS.getDirectiveTokens(TestFile.path()); @@ -53,10 +60,16 @@ TEST(DependencyScanningCASFilesystem, DirectiveScanFailure) { TempDir TestDir("DependencyScanningCASFilesystemTest", /*Unique=*/true); TempFile TestFile(TestDir.path("python"), "", "import sys\n"); - std::unique_ptr CAS = llvm::cas::createInMemoryCAS(); - std::unique_ptr Cache = llvm::cas::createInMemoryActionCache(); + std::shared_ptr CAS = llvm::cas::createInMemoryCAS(); + std::shared_ptr Cache = llvm::cas::createInMemoryActionCache(); auto CacheFS = llvm::cantFail(llvm::cas::createCachingOnDiskFileSystem(*CAS)); - DependencyScanningCASFilesystem FS(CacheFS, *Cache); + DependencyScanningService Service( + ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make, + clang::CASOptions(), CAS, Cache, nullptr, + ScanningOptimizations::Default, /*EagerLoadModules=*/false, + /*TraceVFS=*/false, llvm::sys::toTimeT(std::chrono::system_clock::now()), + true); + DependencyScanningCASFilesystem FS(Service, CacheFS); EXPECT_EQ(FS.status(TestFile.path()).getError(), std::error_code()); auto Directives = FS.getDirectiveTokens(TestFile.path()); @@ -64,7 +77,7 @@ TEST(DependencyScanningCASFilesystem, DirectiveScanFailure) { // Check the cached failure in the action cache. { - DependencyScanningCASFilesystem NewFS(CacheFS, *Cache); + DependencyScanningCASFilesystem NewFS(Service, CacheFS); EXPECT_EQ(NewFS.status(TestFile.path()).getError(), std::error_code()); auto Directives = NewFS.getDirectiveTokens(TestFile.path()); ASSERT_FALSE(Directives);