diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h index 24cf1afaaa450..a54a6269dbdc4 100644 --- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h +++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h @@ -16,7 +16,6 @@ #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Serialization/ObjectFilePCHContainerReader.h" #include "llvm/Support/VirtualFileSystem.h" namespace clang { diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index 0a5c4936f7fe7..92da219d85d56 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -140,20 +140,6 @@ class DependencyScanningWorker { friend tooling::CompilerInstanceWithContext; }; - -std::pair, - std::vector> -initVFSForTUBufferScanning(IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, - StringRef WorkingDirectory, - llvm::MemoryBufferRef TUBuffer); - -std::pair, - std::vector> -initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, - StringRef WorkingDirectory, StringRef ModuleName); - } // end namespace dependencies } // end namespace clang diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h index 40e791fd5bff5..30846ae0ebf3e 100644 --- a/clang/include/clang/Tooling/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -226,6 +226,16 @@ class CompilerInstanceWithContext { StringRef ModuleName, const llvm::DenseSet &AlreadySeen, dependencies::LookupModuleOutputCallback LookupModuleOutput); + + // MaxNumOfQueries is the upper limit of the number of names the by-name + // scanning API (computeDependencies) can support after a + // CompilerInstanceWithContext is initialized. At the time of this commit, the + // estimated number of total unique importable names is around 3000 from + // Apple's SDKs. We usually import them in parallel, so it is unlikely that + // all names are all scanned by the same dependency scanning worker. Therefore + // the 64k (20x bigger than our estimate) size is sufficient to hold the + // unique source locations to report diagnostics per worker. + static const int32_t MaxNumOfQueries = 1 << 16; }; } // end namespace tooling diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index 7a7f34a43b338..75eb821ac651a 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -108,58 +108,3 @@ bool DependencyScanningWorker::computeDependencies( return Success && Action.hasScanned(); } - -std::pair, - std::vector> -dependencies::initVFSForTUBufferScanning( - IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, StringRef WorkingDirectory, - llvm::MemoryBufferRef TUBuffer) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - auto OverlayFS = - llvm::makeIntrusiveRefCnt(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - auto InputPath = TUBuffer.getBufferIdentifier(); - InMemoryFS->addFile( - InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); - IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; - - OverlayFS->pushOverlay(InMemoryOverlay); - std::vector ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(InputPath); - - return std::make_pair(OverlayFS, ModifiedCommandLine); -} - -std::pair, - std::vector> -dependencies::initVFSForByNameScanning( - IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, StringRef WorkingDirectory, - StringRef ModuleName) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - // If we're scanning based on a module name alone, we don't expect the client - // to provide us with an input file. However, the driver really wants to have - // one. Let's just make it up to make the driver happy. - auto OverlayFS = - llvm::makeIntrusiveRefCnt(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - SmallString<128> FakeInputPath; - // TODO: We should retry the creation if the path already exists. - llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath, - /*MakeAbsolute=*/false); - InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); - IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; - OverlayFS->pushOverlay(InMemoryOverlay); - - std::vector ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(FakeInputPath); - - return std::make_pair(OverlayFS, ModifiedCommandLine); -} diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index b570bc8e5ae30..4a013ad86788c 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -253,6 +253,63 @@ std::optional DependencyScanningTool::getP1689ModuleDependencyFile( return Rule; } +static std::pair, + std::vector> +initVFSForTUBufferScanning(IntrusiveRefCntPtr BaseFS, + ArrayRef CommandLine, + StringRef WorkingDirectory, + llvm::MemoryBufferRef TUBuffer) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + auto OverlayFS = + llvm::makeIntrusiveRefCnt(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + auto InputPath = TUBuffer.getBufferIdentifier(); + InMemoryFS->addFile( + InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); + IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; + + OverlayFS->pushOverlay(InMemoryOverlay); + std::vector ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(InputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} + +static std::pair, + std::vector> +initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, + ArrayRef CommandLine, + StringRef WorkingDirectory) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + // If we're scanning based on a module name alone, we don't expect the client + // to provide us with an input file. However, the driver really wants to have + // one. Let's just make it up to make the driver happy. + auto OverlayFS = + llvm::makeIntrusiveRefCnt(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + StringRef FakeInputPath("module-include.input"); + // The fake input buffer is read-only, and it is used to produce + // unique source locations for the diagnostics. Therefore sharing + // this global buffer across threads is ok. + static const std::string FakeInput( + clang::tooling::CompilerInstanceWithContext::MaxNumOfQueries, ' '); + InMemoryFS->addFile(FakeInputPath, 0, + llvm::MemoryBuffer::getMemBuffer(FakeInput)); + IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; + OverlayFS->pushOverlay(InMemoryOverlay); + + std::vector ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(FakeInputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} + std::optional DependencyScanningTool::getTranslationUnitDependencies( ArrayRef CommandLine, StringRef CWD, @@ -320,8 +377,8 @@ std::optional CompilerInstanceWithContext::initializeFromCommandline( DependencyScanningTool &Tool, StringRef CWD, ArrayRef CommandLine, DiagnosticConsumer &DC) { - auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning( - &Tool.Worker.getVFS(), CommandLine, CWD, "ScanningByName"); + auto [OverlayFS, ModifiedCommandLine] = + initVFSForByNameScanning(&Tool.Worker.getVFS(), CommandLine, CWD); auto DiagEngineWithCmdAndOpts = std::make_unique(ModifiedCommandLine, OverlayFS, DC); @@ -446,6 +503,9 @@ bool CompilerInstanceWithContext::initialize( bool CompilerInstanceWithContext::computeDependencies( StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller) { + if (SrcLocOffset >= MaxNumOfQueries) + llvm::report_fatal_error("exceeded maximum by-name scans for worker"); + assert(CIPtr && "CIPtr must be initialized before calling this method"); auto &CI = *CIPtr; diff --git a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c index 9889982354e90..f99690c3a7715 100644 --- a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c +++ b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names-diagnostics.c @@ -27,11 +27,11 @@ module root2 { header "root2.h" } // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s // ERROR: Error while scanning dependencies for modA: -// ERROR-NEXT: {{.*}}: fatal error: module 'modA' not found +// ERROR-NEXT: module-include.input:1:1: fatal error: module 'modA' not found // ERROR-NEXT: Error while scanning dependencies for modB: -// ERROR-NEXT: {{.*}}: fatal error: module 'modB' not found +// ERROR-NEXT: module-include.input:1:3: fatal error: module 'modB' not found // ERROR-NEXT: Error while scanning dependencies for modC: -// ERROR-NEXT: {{.*}}: fatal error: module 'modC' not found +// ERROR-NEXT: module-include.input:1:4: fatal error: module 'modC' not found // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: {