diff --git a/clang-tools-extra/clangd/ProjectModules.cpp b/clang-tools-extra/clangd/ProjectModules.cpp index 33cbc715e9659..4ea38f4177fe9 100644 --- a/clang-tools-extra/clangd/ProjectModules.cpp +++ b/clang-tools-extra/clangd/ProjectModules.cpp @@ -10,6 +10,7 @@ #include "Compiler.h" #include "support/Logger.h" #include "clang/DependencyScanning/DependencyScanningService.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Tooling/DependencyScanningTool.h" #include "clang/Tooling/Tooling.h" #include "llvm/ADT/SmallString.h" diff --git a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h deleted file mode 100644 index dff959a70be8f..0000000000000 --- a/clang/include/clang/DependencyScanning/DependencyScannerImpl.h +++ /dev/null @@ -1,89 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNERIMPL_H -#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNERIMPL_H - -#include "clang/DependencyScanning/DependencyScanningFilesystem.h" -#include "clang/DependencyScanning/ModuleDepCollector.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Frontend/CompilerInvocation.h" -#include "clang/Frontend/TextDiagnosticPrinter.h" -#include "llvm/Support/VirtualFileSystem.h" - -namespace clang { -class DiagnosticConsumer; - -namespace dependencies { -class DependencyScanningService; -class DependencyScanningWorker; - -class DependencyConsumer; -class DependencyActionController; -class DependencyScanningWorkerFilesystem; - -// Helper functions and data types. -std::unique_ptr -createDiagOptions(ArrayRef CommandLine); - -struct DiagnosticsEngineWithDiagOpts { - // We need to bound the lifetime of the DiagOpts used to create the - // DiganosticsEngine with the DiagnosticsEngine itself. - std::unique_ptr DiagOpts; - IntrusiveRefCntPtr DiagEngine; - - DiagnosticsEngineWithDiagOpts(ArrayRef CommandLine, - IntrusiveRefCntPtr FS, - DiagnosticConsumer &DC); -}; - -std::unique_ptr -createCompilerInvocation(ArrayRef CommandLine, - DiagnosticsEngine &Diags); - -/// Canonicalizes command-line macro defines (e.g. removing "-DX -UX"). -void canonicalizeDefines(PreprocessorOptions &PPOpts); - -/// Creates a CompilerInvocation suitable for the dependency scanner. -std::shared_ptr -createScanCompilerInvocation(const CompilerInvocation &Invocation, - const DependencyScanningService &Service, - DependencyActionController &Controller); - -/// Creates dependency output options to be reported to the dependency consumer, -/// deducing missing information if necessary. -std::unique_ptr -createDependencyOutputOptions(const CompilerInvocation &Invocation); - -void initializeScanCompilerInstance( - CompilerInstance &ScanInstance, - IntrusiveRefCntPtr FS, - DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, - IntrusiveRefCntPtr DepFS); - -SmallVector -getInitialStableDirs(const CompilerInstance &ScanInstance); - -std::optional -computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, - SmallVector &StableDirs); - -/// Create the dependency collector that will collect the produced -/// dependencies. May return the created ModuleDepCollector depending -/// on the scanning format. -std::shared_ptr initializeScanInstanceDependencyCollector( - CompilerInstance &ScanInstance, - std::unique_ptr DepOutputOpts, - DependencyScanningService &Service, CompilerInvocation &Inv, - DependencyActionController &Controller, - PrebuiltModulesAttrsMap PrebuiltModulesASTMap, - SmallVector &StableDirs); -} // namespace dependencies -} // namespace clang - -#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNERIMPL_H diff --git a/clang/include/clang/DependencyScanning/DependencyScanningUtils.h b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h index 33f7216762b03..952342d2a322d 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningUtils.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h @@ -11,7 +11,6 @@ #include "clang/DependencyScanning/DependencyActionController.h" #include "clang/DependencyScanning/DependencyConsumer.h" -#include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/DependencyScanning/DependencyScanningWorker.h" #include "clang/DependencyScanning/ModuleDepCollector.h" #include "llvm/ADT/DenseSet.h" diff --git a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h index e30e2280f13af..638c317d9ce64 100644 --- a/clang/include/clang/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -9,11 +9,10 @@ #ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H #define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H -#include "clang/Basic/AtomicLineLogger.h" +#include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticOptions.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/LLVM.h" -#include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/DependencyScanning/DependencyScanningService.h" #include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Frontend/PCHContainerOperations.h" @@ -21,6 +20,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/VirtualFileSystem.h" +#include #include #include @@ -33,6 +33,9 @@ namespace dependencies { class DependencyConsumer; class DependencyScanningWorkerFilesystem; +std::unique_ptr +createScanningDiagOptions(ArrayRef CommandLine); + /// An individual dependency scanning worker that is able to run on its own /// thread. /// diff --git a/clang/include/clang/Tooling/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h index 9befa4858eba6..53a6615ab88c1 100644 --- a/clang/include/clang/Tooling/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -9,7 +9,6 @@ #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H -#include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/DependencyScanning/DependencyScanningService.h" #include "clang/DependencyScanning/DependencyScanningUtils.h" #include "clang/DependencyScanning/DependencyScanningWorker.h" diff --git a/clang/lib/DependencyScanning/CMakeLists.txt b/clang/lib/DependencyScanning/CMakeLists.txt index 015c6bcb12326..88eb6408842f9 100644 --- a/clang/lib/DependencyScanning/CMakeLists.txt +++ b/clang/lib/DependencyScanning/CMakeLists.txt @@ -11,7 +11,6 @@ add_clang_library(clangDependencyScanning DependencyScanningService.cpp DependencyScanningWorker.cpp DependencyScanningUtils.cpp - DependencyScannerImpl.cpp InProcessModuleCache.cpp ModuleDepCollector.cpp diff --git a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp deleted file mode 100644 index c636da996cb71..0000000000000 --- a/clang/lib/DependencyScanning/DependencyScannerImpl.cpp +++ /dev/null @@ -1,525 +0,0 @@ -//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "clang/DependencyScanning/DependencyScannerImpl.h" -#include "clang/Basic/DiagnosticFrontend.h" -#include "clang/Basic/DiagnosticSerialization.h" -#include "clang/DependencyScanning/DependencyActionController.h" -#include "clang/DependencyScanning/DependencyConsumer.h" -#include "clang/DependencyScanning/DependencyScanningFilesystem.h" -#include "clang/DependencyScanning/DependencyScanningService.h" -#include "clang/DependencyScanning/DependencyScanningWorker.h" -#include "clang/Frontend/FrontendActions.h" -#include "llvm/ADT/IntrusiveRefCntPtr.h" -#include "llvm/ADT/ScopeExit.h" -#include "llvm/Option/Option.h" -#include "llvm/Support/VirtualFileSystem.h" -#include "llvm/TargetParser/Host.h" - -using namespace clang; -using namespace dependencies; - -static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts, - const HeaderSearchOptions &ExistingHSOpts, - DiagnosticsEngine *Diags, - const LangOptions &LangOpts) { - if (LangOpts.Modules) { - if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) { - if (Diags) { - Diags->Report(diag::warn_pch_vfsoverlay_mismatch); - auto VFSNote = [&](int Type, ArrayRef VFSOverlays) { - if (VFSOverlays.empty()) { - Diags->Report(diag::note_pch_vfsoverlay_empty) << Type; - } else { - std::string Files = llvm::join(VFSOverlays, "\n"); - Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files; - } - }; - VFSNote(0, HSOpts.VFSOverlayFiles); - VFSNote(1, ExistingHSOpts.VFSOverlayFiles); - } - } - } - return false; -} - -namespace { - -using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); - -/// A listener that collects the imported modules and the input -/// files. While visiting, collect vfsoverlays and file inputs that determine -/// whether prebuilt modules fully resolve in stable directories. -class PrebuiltModuleListener : public ASTReaderListener { -public: - PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles, - llvm::SmallVector &NewModuleFiles, - PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, - const HeaderSearchOptions &HSOpts, - const LangOptions &LangOpts, DiagnosticsEngine &Diags, - const ArrayRef StableDirs) - : PrebuiltModuleFiles(PrebuiltModuleFiles), - NewModuleFiles(NewModuleFiles), - PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts), - ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {} - - bool needsImportVisitation() const override { return true; } - bool needsInputFileVisitation() override { return true; } - bool needsSystemInputFileVisitation() override { return true; } - - /// Accumulate the modules are transitively depended on by the initial - /// prebuilt module. - void visitImport(StringRef ModuleName, StringRef Filename) override { - if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second) - NewModuleFiles.push_back(Filename.str()); - - auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename); - PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; - if (PrebuiltMapEntry.second) - PrebuiltModule.setInStableDir(!StableDirs.empty()); - - if (auto It = PrebuiltModulesASTMap.find(CurrentFile); - It != PrebuiltModulesASTMap.end() && CurrentFile != Filename) - PrebuiltModule.addDependent(It->getKey()); - } - - /// For each input file discovered, check whether it's external path is in a - /// stable directory. Traversal is stopped if the current module is not - /// considered stable. - bool visitInputFileAsRequested(StringRef FilenameAsRequested, - StringRef Filename, bool isSystem, - bool isOverridden, time_t StoredTime, - bool isExplicitModule) override { - if (StableDirs.empty()) - return false; - auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); - if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) || - (!PrebuiltEntryIt->second.isInStableDir())) - return false; - - PrebuiltEntryIt->second.setInStableDir( - isPathInStableDir(StableDirs, Filename)); - return PrebuiltEntryIt->second.isInStableDir(); - } - - /// Update which module that is being actively traversed. - void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind, - bool DirectlyImported) override { - // If the CurrentFile is not - // considered stable, update any of it's transitive dependents. - auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); - if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) && - !PrebuiltEntryIt->second.isInStableDir()) - PrebuiltEntryIt->second.updateDependentsNotInStableDirs( - PrebuiltModulesASTMap); - CurrentFile = Filename.str(); - } - - /// Check the header search options for a given module when considering - /// if the module comes from stable directories. - bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, - StringRef ModuleFilename, StringRef ContextHash, - bool Complain) override { - - auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); - PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; - if (PrebuiltMapEntry.second) - PrebuiltModule.setInStableDir(!StableDirs.empty()); - - if (PrebuiltModule.isInStableDir()) - PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts)); - - return false; - } - - /// Accumulate vfsoverlays used to build these prebuilt modules. - bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts, - bool Complain) override { - - auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); - PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; - if (PrebuiltMapEntry.second) - PrebuiltModule.setInStableDir(!StableDirs.empty()); - - PrebuiltModule.setVFS( - llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles)); - - return checkHeaderSearchPaths( - HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts); - } - -private: - PrebuiltModuleFilesT &PrebuiltModuleFiles; - llvm::SmallVector &NewModuleFiles; - PrebuiltModulesAttrsMap &PrebuiltModulesASTMap; - const HeaderSearchOptions &ExistingHSOpts; - const LangOptions &ExistingLangOpts; - DiagnosticsEngine &Diags; - std::string CurrentFile; - const ArrayRef StableDirs; -}; - -/// Visit the given prebuilt module and collect all of the modules it -/// transitively imports and contributing input files. -static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, - CompilerInstance &CI, - PrebuiltModuleFilesT &ModuleFiles, - PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, - DiagnosticsEngine &Diags, - const ArrayRef StableDirs) { - // List of module files to be processed. - llvm::SmallVector Worklist; - - PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap, - CI.getHeaderSearchOpts(), CI.getLangOpts(), - Diags, StableDirs); - - Listener.visitModuleFile(ModuleFileName::makeExplicit(PrebuiltModuleFilename), - serialization::MK_ExplicitModule, - /*DirectlyImported=*/true); - if (ASTReader::readASTFileControlBlock( - PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(), - CI.getPCHContainerReader(), - /*FindModuleFileExtensions=*/false, Listener, - /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate)) - return true; - - while (!Worklist.empty()) { - // FIXME: This is assuming the PCH only refers to explicitly-built modules, - // which technically is not guaranteed. To remove the assumption, we'd need - // to also rework how the module files are handled to the scan, specifically - // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain - // paths to ModuleFileName. - Listener.visitModuleFile(ModuleFileName::makeExplicit(Worklist.back()), - serialization::MK_ExplicitModule, - /*DirectlyImported=*/false); - if (ASTReader::readASTFileControlBlock( - Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(), - CI.getPCHContainerReader(), - /*FindModuleFileExtensions=*/false, Listener, - /*ValidateDiagnosticOptions=*/false)) - return true; - } - return false; -} - -/// Transform arbitrary file name into an object-like file name. -static std::string makeObjFileName(StringRef FileName) { - SmallString<128> ObjFileName(FileName); - llvm::sys::path::replace_extension(ObjFileName, "o"); - return std::string(ObjFileName); -} - -/// Deduce the dependency target based on the output file and input files. -static std::string -deduceDepTarget(const std::string &OutputFile, - const SmallVectorImpl &InputFiles) { - if (OutputFile != "-") - return OutputFile; - - if (InputFiles.empty() || !InputFiles.front().isFile()) - return "clang-scan-deps\\ dependency"; - - return makeObjFileName(InputFiles.front().getFile()); -} - -// Clang implements -D and -U by splatting text into a predefines buffer. This -// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and -// define the same macro, or adding C++ style comments before the macro name. -// -// This function checks that the first non-space characters in the macro -// obviously form an identifier that can be uniqued on without lexing. Failing -// to do this could lead to changing the final definition of a macro. -// -// We could set up a preprocessor and actually lex the name, but that's very -// heavyweight for a situation that will almost never happen in practice. -static std::optional getSimpleMacroName(StringRef Macro) { - StringRef Name = Macro.split("=").first.ltrim(" \t"); - std::size_t I = 0; - - auto FinishName = [&]() -> std::optional { - StringRef SimpleName = Name.slice(0, I); - if (SimpleName.empty()) - return std::nullopt; - return SimpleName; - }; - - for (; I != Name.size(); ++I) { - switch (Name[I]) { - case '(': // Start of macro parameter list - case ' ': // End of macro name - case '\t': - return FinishName(); - case '_': - continue; - default: - if (llvm::isAlnum(Name[I])) - continue; - return std::nullopt; - } - } - return FinishName(); -} -} // namespace - -void dependencies::canonicalizeDefines(PreprocessorOptions &PPOpts) { - using MacroOpt = std::pair; - std::vector SimpleNames; - SimpleNames.reserve(PPOpts.Macros.size()); - std::size_t Index = 0; - for (const auto &M : PPOpts.Macros) { - auto SName = getSimpleMacroName(M.first); - // Skip optimizing if we can't guarantee we can preserve relative order. - if (!SName) - return; - SimpleNames.emplace_back(*SName, Index); - ++Index; - } - - llvm::stable_sort(SimpleNames, llvm::less_first()); - // Keep the last instance of each macro name by going in reverse - auto NewEnd = std::unique( - SimpleNames.rbegin(), SimpleNames.rend(), - [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; }); - SimpleNames.erase(SimpleNames.begin(), NewEnd.base()); - - // Apply permutation. - decltype(PPOpts.Macros) NewMacros; - NewMacros.reserve(SimpleNames.size()); - for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) { - std::size_t OriginalIndex = SimpleNames[I].second; - // We still emit undefines here as they may be undefining a predefined macro - NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex])); - } - std::swap(PPOpts.Macros, NewMacros); -} - -namespace { -class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter { - DependencyScanningWorkerFilesystem *DepFS; - -public: - ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) { - FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) { - auto *DFS = llvm::dyn_cast(&FS); - if (DFS) { - assert(!DepFS && "Found multiple scanning VFSs"); - DepFS = DFS; - } - }); - assert(DepFS && "Did not find scanning VFS"); - } - - std::unique_ptr - cloneFor(FileManager &FileMgr) override { - return std::make_unique(FileMgr); - } - - std::optional> - operator()(FileEntryRef File) override { - return DepFS->getDirectiveTokens(File.getName()); - } -}; - -/// Sanitize diagnostic options for dependency scan. -void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) { - // Don't print 'X warnings and Y errors generated'. - DiagOpts.ShowCarets = false; - // Don't write out diagnostic file. - DiagOpts.DiagnosticSerializationFile.clear(); - // Don't emit warnings except for scanning specific warnings. - // TODO: It would be useful to add a more principled way to ignore all - // warnings that come from source code. The issue is that we need to - // ignore warnings that could be surpressed by - // `#pragma clang diagnostic`, while still allowing some scanning - // warnings for things we're not ready to turn into errors yet. - // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example. - llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) { - return llvm::StringSwitch(Warning) - .Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false) - .StartsWith("no-error=", false) - .Default(true); - }); -} -} // namespace - -std::unique_ptr -dependencies::createDiagOptions(ArrayRef CommandLine) { - std::vector CLI; - for (const std::string &Arg : CommandLine) - CLI.push_back(Arg.c_str()); - auto DiagOpts = CreateAndPopulateDiagOpts(CLI); - sanitizeDiagOpts(*DiagOpts); - return DiagOpts; -} - -DiagnosticsEngineWithDiagOpts::DiagnosticsEngineWithDiagOpts( - ArrayRef CommandLine, - IntrusiveRefCntPtr FS, DiagnosticConsumer &DC) { - std::vector CCommandLine(CommandLine.size(), nullptr); - llvm::transform(CommandLine, CCommandLine.begin(), - [](const std::string &Str) { return Str.c_str(); }); - DiagOpts = CreateAndPopulateDiagOpts(CCommandLine); - sanitizeDiagOpts(*DiagOpts); - DiagEngine = CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DC, - /*ShouldOwnClient=*/false); -} - -std::unique_ptr -dependencies::createCompilerInvocation(ArrayRef CommandLine, - DiagnosticsEngine &Diags) { - llvm::opt::ArgStringList Argv; - for (const std::string &Str : ArrayRef(CommandLine).drop_front()) - Argv.push_back(Str.c_str()); - - auto Invocation = std::make_unique(); - if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) { - // FIXME: Should we just go on like cc1_main does? - return nullptr; - } - return Invocation; -} - -void dependencies::initializeScanCompilerInstance( - CompilerInstance &ScanInstance, - IntrusiveRefCntPtr FS, - DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, - IntrusiveRefCntPtr DepFS) { - ScanInstance.setBuildingModule(false); - ScanInstance.createVirtualFileSystem(FS, DiagConsumer); - ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); - if (!Service.getOpts().EmitWarnings) - ScanInstance.getDiagnostics().setIgnoreAllWarnings(true); - ScanInstance.createFileManager(); - ScanInstance.createSourceManager(); - - // Use DepFS for getting the dependency directives if requested to do so. - if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) - ScanInstance.setDependencyDirectivesGetter( - std::make_unique( - ScanInstance.getFileManager())); -} - -std::shared_ptr dependencies::createScanCompilerInvocation( - const CompilerInvocation &Invocation, - const DependencyScanningService &Service, - DependencyActionController &Controller) { - auto ScanInvocation = std::make_shared(Invocation); - - sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts()); - - ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = - true; - - if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession) - ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp = - Service.getOpts().BuildSessionTimestamp; - - ScanInvocation->getFrontendOpts().DisableFree = false; - ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false; - ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false; - ScanInvocation->getFrontendOpts().GenReducedBMI = false; - ScanInvocation->getFrontendOpts().ModuleOutputPath.clear(); - // This will prevent us compiling individual modules asynchronously since - // FileManager is not thread-safe, but it does improve performance for now. - ScanInvocation->getFrontendOpts().ModulesShareFileManager = true; - ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw"; - ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage = - any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS); - - // Consider different header search and diagnostic options to create - // different modules. This avoids the unsound aliasing of module PCMs. - // - // TODO: Implement diagnostic bucketing to reduce the impact of strict - // context hashing. - ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true; - ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true; - ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; - ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; - ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = - true; - ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false; - - // FIXME: Do this even with PCHs by marking the option as something like - // "preprocessor benign" in LangOptions.def so that it passes the - // compatibility checks in ASTReader. - if (ScanInvocation->getPreprocessorOpts().ImplicitPCHInclude.empty()) { - // Application extension only affects the handling of availability - // attributes, which cannot change the dependencies. - ScanInvocation->getLangOpts().AppExt = false; - } - - // Ensure that the scanner does not create new dependency collectors, - // and thus won't write out the extra '.d' files to disk. - ScanInvocation->getDependencyOutputOpts() = {}; - - Controller.initializeScanInvocation(*ScanInvocation); - - return ScanInvocation; -} - -llvm::SmallVector -dependencies::getInitialStableDirs(const CompilerInstance &ScanInstance) { - // Create a collection of stable directories derived from the ScanInstance - // for determining whether module dependencies would fully resolve from - // those directories. - llvm::SmallVector StableDirs; - const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot; - if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot)) - StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir}; - return StableDirs; -} - -std::optional -dependencies::computePrebuiltModulesASTMap( - CompilerInstance &ScanInstance, llvm::SmallVector &StableDirs) { - // Store a mapping of prebuilt module files and their properties like header - // search options. This will prevent the implicit build to create duplicate - // modules and will force reuse of the existing prebuilt module files - // instead. - PrebuiltModulesAttrsMap PrebuiltModulesASTMap; - - if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) - if (visitPrebuiltModule( - ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, - ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, - PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs)) - return {}; - - return PrebuiltModulesASTMap; -} - -std::unique_ptr -dependencies::createDependencyOutputOptions( - const CompilerInvocation &Invocation) { - auto Opts = std::make_unique( - Invocation.getDependencyOutputOpts()); - // We need at least one -MT equivalent for the generator of make dependency - // files to work. - if (Opts->Targets.empty()) - Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile, - Invocation.getFrontendOpts().Inputs)}; - Opts->IncludeSystemHeaders = true; - - return Opts; -} - -std::shared_ptr -dependencies::initializeScanInstanceDependencyCollector( - CompilerInstance &ScanInstance, - std::unique_ptr DepOutputOpts, - DependencyScanningService &Service, CompilerInvocation &Inv, - DependencyActionController &Controller, - PrebuiltModulesAttrsMap PrebuiltModulesASTMap, - SmallVector &StableDirs) { - auto MDC = std::make_shared( - Service, std::move(DepOutputOpts), ScanInstance, Controller, Inv, - std::move(PrebuiltModulesASTMap), StableDirs); - ScanInstance.addDependencyCollector(MDC); - return MDC; -} diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp index b7fb7de6a1040..c681bb93b8314 100644 --- a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -9,23 +9,502 @@ #include "clang/DependencyScanning/DependencyScanningWorker.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticFrontend.h" +#include "clang/Basic/DiagnosticSerialization.h" #include "clang/DependencyScanning/DependencyActionController.h" #include "clang/DependencyScanning/DependencyConsumer.h" -#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/DependencyScanning/DependencyScanningFilesystem.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Lex/Preprocessor.h" +#include "clang/Sema/SemaOpenACC.h" #include "clang/Serialization/ObjectFilePCHContainerReader.h" #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/ScopeExit.h" +#include "llvm/Option/Option.h" #include "llvm/Support/AdvisoryLock.h" #include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/VirtualFileSystem.h" +#include "llvm/TargetParser/Host.h" #include #include using namespace clang; using namespace dependencies; +static bool checkHeaderSearchPaths(const HeaderSearchOptions &HSOpts, + const HeaderSearchOptions &ExistingHSOpts, + DiagnosticsEngine *Diags, + const LangOptions &LangOpts) { + if (LangOpts.Modules) { + if (HSOpts.VFSOverlayFiles != ExistingHSOpts.VFSOverlayFiles) { + if (Diags) { + Diags->Report(diag::warn_pch_vfsoverlay_mismatch); + auto VFSNote = [&](int Type, ArrayRef VFSOverlays) { + if (VFSOverlays.empty()) { + Diags->Report(diag::note_pch_vfsoverlay_empty) << Type; + } else { + std::string Files = llvm::join(VFSOverlays, "\n"); + Diags->Report(diag::note_pch_vfsoverlay_files) << Type << Files; + } + }; + VFSNote(0, HSOpts.VFSOverlayFiles); + VFSNote(1, ExistingHSOpts.VFSOverlayFiles); + } + } + } + return false; +} +namespace { +using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles); + +/// A listener that collects the imported modules and the input +/// files. While visiting, collect vfsoverlays and file inputs that determine +/// whether prebuilt modules fully resolve in stable directories. +class PrebuiltModuleListener : public ASTReaderListener { +public: + PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles, + llvm::SmallVector &NewModuleFiles, + PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, + const HeaderSearchOptions &HSOpts, + const LangOptions &LangOpts, DiagnosticsEngine &Diags, + const ArrayRef StableDirs) + : PrebuiltModuleFiles(PrebuiltModuleFiles), + NewModuleFiles(NewModuleFiles), + PrebuiltModulesASTMap(PrebuiltModulesASTMap), ExistingHSOpts(HSOpts), + ExistingLangOpts(LangOpts), Diags(Diags), StableDirs(StableDirs) {} + + bool needsImportVisitation() const override { return true; } + bool needsInputFileVisitation() override { return true; } + bool needsSystemInputFileVisitation() override { return true; } + + /// Accumulate the modules are transitively depended on by the initial + /// prebuilt module. + void visitImport(StringRef ModuleName, StringRef Filename) override { + if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second) + NewModuleFiles.push_back(Filename.str()); + + auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(Filename); + PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; + if (PrebuiltMapEntry.second) + PrebuiltModule.setInStableDir(!StableDirs.empty()); + + if (auto It = PrebuiltModulesASTMap.find(CurrentFile); + It != PrebuiltModulesASTMap.end() && CurrentFile != Filename) + PrebuiltModule.addDependent(It->getKey()); + } + + /// For each input file discovered, check whether it's external path is in a + /// stable directory. Traversal is stopped if the current module is not + /// considered stable. + bool visitInputFileAsRequested(StringRef FilenameAsRequested, + StringRef Filename, bool isSystem, + bool isOverridden, time_t StoredTime, + bool isExplicitModule) override { + if (StableDirs.empty()) + return false; + auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); + if ((PrebuiltEntryIt == PrebuiltModulesASTMap.end()) || + (!PrebuiltEntryIt->second.isInStableDir())) + return false; + + PrebuiltEntryIt->second.setInStableDir( + isPathInStableDir(StableDirs, Filename)); + return PrebuiltEntryIt->second.isInStableDir(); + } + + /// Update which module that is being actively traversed. + void visitModuleFile(ModuleFileName Filename, serialization::ModuleKind Kind, + bool DirectlyImported) override { + // If the CurrentFile is not + // considered stable, update any of it's transitive dependents. + auto PrebuiltEntryIt = PrebuiltModulesASTMap.find(CurrentFile); + if ((PrebuiltEntryIt != PrebuiltModulesASTMap.end()) && + !PrebuiltEntryIt->second.isInStableDir()) + PrebuiltEntryIt->second.updateDependentsNotInStableDirs( + PrebuiltModulesASTMap); + CurrentFile = Filename.str(); + } + + /// Check the header search options for a given module when considering + /// if the module comes from stable directories. + bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts, + StringRef ModuleFilename, StringRef ContextHash, + bool Complain) override { + + auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); + PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; + if (PrebuiltMapEntry.second) + PrebuiltModule.setInStableDir(!StableDirs.empty()); + + if (PrebuiltModule.isInStableDir()) + PrebuiltModule.setInStableDir(areOptionsInStableDir(StableDirs, HSOpts)); + + return false; + } + + /// Accumulate vfsoverlays used to build these prebuilt modules. + bool ReadHeaderSearchPaths(const HeaderSearchOptions &HSOpts, + bool Complain) override { + + auto PrebuiltMapEntry = PrebuiltModulesASTMap.try_emplace(CurrentFile); + PrebuiltModuleASTAttrs &PrebuiltModule = PrebuiltMapEntry.first->second; + if (PrebuiltMapEntry.second) + PrebuiltModule.setInStableDir(!StableDirs.empty()); + + PrebuiltModule.setVFS( + llvm::StringSet<>(llvm::from_range, HSOpts.VFSOverlayFiles)); + + return checkHeaderSearchPaths( + HSOpts, ExistingHSOpts, Complain ? &Diags : nullptr, ExistingLangOpts); + } + +private: + PrebuiltModuleFilesT &PrebuiltModuleFiles; + llvm::SmallVector &NewModuleFiles; + PrebuiltModulesAttrsMap &PrebuiltModulesASTMap; + const HeaderSearchOptions &ExistingHSOpts; + const LangOptions &ExistingLangOpts; + DiagnosticsEngine &Diags; + std::string CurrentFile; + const ArrayRef StableDirs; +}; +} // namespace + +/// Visit the given prebuilt module and collect all of the modules it +/// transitively imports and contributing input files. +static bool visitPrebuiltModule(StringRef PrebuiltModuleFilename, + CompilerInstance &CI, + PrebuiltModuleFilesT &ModuleFiles, + PrebuiltModulesAttrsMap &PrebuiltModulesASTMap, + DiagnosticsEngine &Diags, + const ArrayRef StableDirs) { + // List of module files to be processed. + llvm::SmallVector Worklist; + + PrebuiltModuleListener Listener(ModuleFiles, Worklist, PrebuiltModulesASTMap, + CI.getHeaderSearchOpts(), CI.getLangOpts(), + Diags, StableDirs); + + Listener.visitModuleFile(ModuleFileName::makeExplicit(PrebuiltModuleFilename), + serialization::MK_ExplicitModule, + /*DirectlyImported=*/true); + if (ASTReader::readASTFileControlBlock( + PrebuiltModuleFilename, CI.getFileManager(), CI.getModuleCache(), + CI.getPCHContainerReader(), + /*FindModuleFileExtensions=*/false, Listener, + /*ValidateDiagnosticOptions=*/false, ASTReader::ARR_OutOfDate)) + return true; + + while (!Worklist.empty()) { + // FIXME: This is assuming the PCH only refers to explicitly-built modules, + // which technically is not guaranteed. To remove the assumption, we'd need + // to also rework how the module files are handled to the scan, specifically + // change the values of HeaderSearchOptions::PrebuiltModuleFiles from plain + // paths to ModuleFileName. + Listener.visitModuleFile(ModuleFileName::makeExplicit(Worklist.back()), + serialization::MK_ExplicitModule, + /*DirectlyImported=*/false); + if (ASTReader::readASTFileControlBlock( + Worklist.pop_back_val(), CI.getFileManager(), CI.getModuleCache(), + CI.getPCHContainerReader(), + /*FindModuleFileExtensions=*/false, Listener, + /*ValidateDiagnosticOptions=*/false)) + return true; + } + return false; +} + +/// Transform arbitrary file name into an object-like file name. +static std::string makeObjFileName(StringRef FileName) { + SmallString<128> ObjFileName(FileName); + llvm::sys::path::replace_extension(ObjFileName, "o"); + return std::string(ObjFileName); +} + +/// Deduce the dependency target based on the output file and input files. +static std::string +deduceDepTarget(const std::string &OutputFile, + const SmallVectorImpl &InputFiles) { + if (OutputFile != "-") + return OutputFile; + + if (InputFiles.empty() || !InputFiles.front().isFile()) + return "clang-scan-deps\\ dependency"; + + return makeObjFileName(InputFiles.front().getFile()); +} + +// Clang implements -D and -U by splatting text into a predefines buffer. This +// allows constructs such as `-DFඞ=3 "-D F\u{0D9E} 4 3 2”` to be accepted and +// define the same macro, or adding C++ style comments before the macro name. +// +// This function checks that the first non-space characters in the macro +// obviously form an identifier that can be uniqued on without lexing. Failing +// to do this could lead to changing the final definition of a macro. +// +// We could set up a preprocessor and actually lex the name, but that's very +// heavyweight for a situation that will almost never happen in practice. +static std::optional getSimpleMacroName(StringRef Macro) { + StringRef Name = Macro.split("=").first.ltrim(" \t"); + std::size_t I = 0; + + auto FinishName = [&]() -> std::optional { + StringRef SimpleName = Name.slice(0, I); + if (SimpleName.empty()) + return std::nullopt; + return SimpleName; + }; + + for (; I != Name.size(); ++I) { + switch (Name[I]) { + case '(': // Start of macro parameter list + case ' ': // End of macro name + case '\t': + return FinishName(); + case '_': + continue; + default: + if (llvm::isAlnum(Name[I])) + continue; + return std::nullopt; + } + } + return FinishName(); +} + +static void canonicalizeDefines(PreprocessorOptions &PPOpts) { + using MacroOpt = std::pair; + std::vector SimpleNames; + SimpleNames.reserve(PPOpts.Macros.size()); + std::size_t Index = 0; + for (const auto &M : PPOpts.Macros) { + auto SName = getSimpleMacroName(M.first); + // Skip optimizing if we can't guarantee we can preserve relative order. + if (!SName) + return; + SimpleNames.emplace_back(*SName, Index); + ++Index; + } + + llvm::stable_sort(SimpleNames, llvm::less_first()); + // Keep the last instance of each macro name by going in reverse + auto NewEnd = std::unique( + SimpleNames.rbegin(), SimpleNames.rend(), + [](const MacroOpt &A, const MacroOpt &B) { return A.first == B.first; }); + SimpleNames.erase(SimpleNames.begin(), NewEnd.base()); + + // Apply permutation. + decltype(PPOpts.Macros) NewMacros; + NewMacros.reserve(SimpleNames.size()); + for (std::size_t I = 0, E = SimpleNames.size(); I != E; ++I) { + std::size_t OriginalIndex = SimpleNames[I].second; + // We still emit undefines here as they may be undefining a predefined macro + NewMacros.push_back(std::move(PPOpts.Macros[OriginalIndex])); + } + std::swap(PPOpts.Macros, NewMacros); +} + +namespace { +class ScanningDependencyDirectivesGetter : public DependencyDirectivesGetter { + DependencyScanningWorkerFilesystem *DepFS; + +public: + ScanningDependencyDirectivesGetter(FileManager &FileMgr) : DepFS(nullptr) { + FileMgr.getVirtualFileSystem().visit([&](llvm::vfs::FileSystem &FS) { + auto *DFS = llvm::dyn_cast(&FS); + if (DFS) { + assert(!DepFS && "Found multiple scanning VFSs"); + DepFS = DFS; + } + }); + assert(DepFS && "Did not find scanning VFS"); + } + + std::unique_ptr + cloneFor(FileManager &FileMgr) override { + return std::make_unique(FileMgr); + } + + std::optional> + operator()(FileEntryRef File) override { + return DepFS->getDirectiveTokens(File.getName()); + } +}; +} // namespace + +/// Sanitize diagnostic options for dependency scan. +static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) { + // Don't print 'X warnings and Y errors generated'. + DiagOpts.ShowCarets = false; + // Don't write out diagnostic file. + DiagOpts.DiagnosticSerializationFile.clear(); + // Don't emit warnings except for scanning specific warnings. + // TODO: It would be useful to add a more principled way to ignore all + // warnings that come from source code. The issue is that we need to + // ignore warnings that could be surpressed by + // `#pragma clang diagnostic`, while still allowing some scanning + // warnings for things we're not ready to turn into errors yet. + // See `test/ClangScanDeps/diagnostic-pragmas.c` for an example. + llvm::erase_if(DiagOpts.Warnings, [](StringRef Warning) { + return llvm::StringSwitch(Warning) + .Cases({"pch-vfs-diff", "error=pch-vfs-diff"}, false) + .StartsWith("no-error=", false) + .Default(true); + }); +} + +static std::unique_ptr +createCompilerInvocation(ArrayRef CommandLine, + DiagnosticsEngine &Diags) { + llvm::opt::ArgStringList Argv; + for (const std::string &Str : ArrayRef(CommandLine).drop_front()) + Argv.push_back(Str.c_str()); + + auto Invocation = std::make_unique(); + if (!CompilerInvocation::CreateFromArgs(*Invocation, Argv, Diags)) { + // FIXME: Should we just go on like cc1_main does? + return nullptr; + } + return Invocation; +} + +static void initializeScanCompilerInstance( + CompilerInstance &ScanInstance, + IntrusiveRefCntPtr FS, + DiagnosticConsumer *DiagConsumer, DependencyScanningService &Service, + IntrusiveRefCntPtr DepFS) { + ScanInstance.setBuildingModule(false); + ScanInstance.createVirtualFileSystem(FS, DiagConsumer); + ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); + if (!Service.getOpts().EmitWarnings) + ScanInstance.getDiagnostics().setIgnoreAllWarnings(true); + ScanInstance.createFileManager(); + ScanInstance.createSourceManager(); + + // Use DepFS for getting the dependency directives if requested to do so. + if (Service.getOpts().Mode == ScanningMode::DependencyDirectivesScan) + ScanInstance.setDependencyDirectivesGetter( + std::make_unique( + ScanInstance.getFileManager())); +} + +static std::shared_ptr +createScanCompilerInvocation(const CompilerInvocation &Invocation, + const DependencyScanningService &Service, + DependencyActionController &Controller) { + auto ScanInvocation = std::make_shared(Invocation); + + sanitizeDiagOpts(ScanInvocation->getDiagnosticOpts()); + + ScanInvocation->getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath = + true; + + if (ScanInvocation->getHeaderSearchOpts().ModulesValidateOncePerBuildSession) + ScanInvocation->getHeaderSearchOpts().BuildSessionTimestamp = + Service.getOpts().BuildSessionTimestamp; + + ScanInvocation->getFrontendOpts().DisableFree = false; + ScanInvocation->getFrontendOpts().GenerateGlobalModuleIndex = false; + ScanInvocation->getFrontendOpts().UseGlobalModuleIndex = false; + ScanInvocation->getFrontendOpts().GenReducedBMI = false; + ScanInvocation->getFrontendOpts().ModuleOutputPath.clear(); + // This will prevent us compiling individual modules asynchronously since + // FileManager is not thread-safe, but it does improve performance for now. + ScanInvocation->getFrontendOpts().ModulesShareFileManager = true; + ScanInvocation->getHeaderSearchOpts().ModuleFormat = "raw"; + ScanInvocation->getHeaderSearchOpts().ModulesIncludeVFSUsage = + any(Service.getOpts().OptimizeArgs & ScanningOptimizations::VFS); + + // Consider different header search and diagnostic options to create + // different modules. This avoids the unsound aliasing of module PCMs. + // + // TODO: Implement diagnostic bucketing to reduce the impact of strict + // context hashing. + ScanInvocation->getHeaderSearchOpts().ModulesStrictContextHash = true; + ScanInvocation->getHeaderSearchOpts().ModulesSerializeOnlyPreprocessor = true; + ScanInvocation->getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; + ScanInvocation->getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; + ScanInvocation->getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = + true; + ScanInvocation->getHeaderSearchOpts().ModulesForceValidateUserHeaders = false; + + // FIXME: Do this even with PCHs by marking the option as something like + // "preprocessor benign" in LangOptions.def so that it passes the + // compatibility checks in ASTReader. + if (ScanInvocation->getPreprocessorOpts().ImplicitPCHInclude.empty()) { + // Application extension only affects the handling of availability + // attributes, which cannot change the dependencies. + ScanInvocation->getLangOpts().AppExt = false; + } + + // Ensure that the scanner does not create new dependency collectors, + // and thus won't write out the extra '.d' files to disk. + ScanInvocation->getDependencyOutputOpts() = {}; + + Controller.initializeScanInvocation(*ScanInvocation); + + return ScanInvocation; +} + +static llvm::SmallVector +getInitialStableDirs(const CompilerInstance &ScanInstance) { + // Create a collection of stable directories derived from the ScanInstance + // for determining whether module dependencies would fully resolve from + // those directories. + llvm::SmallVector StableDirs; + const StringRef Sysroot = ScanInstance.getHeaderSearchOpts().Sysroot; + if (!Sysroot.empty() && (llvm::sys::path::root_directory(Sysroot) != Sysroot)) + StableDirs = {Sysroot, ScanInstance.getHeaderSearchOpts().ResourceDir}; + return StableDirs; +} + +static std::optional +computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, + llvm::SmallVector &StableDirs) { + // Store a mapping of prebuilt module files and their properties like header + // search options. This will prevent the implicit build to create duplicate + // modules and will force reuse of the existing prebuilt module files + // instead. + PrebuiltModulesAttrsMap PrebuiltModulesASTMap; + + if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty()) + if (visitPrebuiltModule( + ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance, + ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles, + PrebuiltModulesASTMap, ScanInstance.getDiagnostics(), StableDirs)) + return {}; + + return PrebuiltModulesASTMap; +} + +static std::unique_ptr +createDependencyOutputOptions(const CompilerInvocation &Invocation) { + auto Opts = std::make_unique( + Invocation.getDependencyOutputOpts()); + // We need at least one -MT equivalent for the generator of make dependency + // files to work. + if (Opts->Targets.empty()) + Opts->Targets = {deduceDepTarget(Invocation.getFrontendOpts().OutputFile, + Invocation.getFrontendOpts().Inputs)}; + Opts->IncludeSystemHeaders = true; + + return Opts; +} + +static std::shared_ptr +initializeScanInstanceDependencyCollector( + CompilerInstance &ScanInstance, + std::unique_ptr DepOutputOpts, + DependencyScanningService &Service, CompilerInvocation &Inv, + DependencyActionController &Controller, + PrebuiltModulesAttrsMap PrebuiltModulesASTMap, + SmallVector &StableDirs) { + auto MDC = std::make_shared( + Service, std::move(DepOutputOpts), ScanInstance, Controller, Inv, + std::move(PrebuiltModulesASTMap), StableDirs); + ScanInstance.addDependencyCollector(MDC); + return MDC; +} + namespace { /// Manages (and terminates) the asynchronous compilation of modules. class AsyncModuleCompiles { @@ -207,6 +686,17 @@ static void runTUModulePrescan(CompilerInstance &PrescanCI, namespace clang { namespace dependencies { + +std::unique_ptr +createScanningDiagOptions(ArrayRef CommandLine) { + std::vector CCommandLine(CommandLine.size(), nullptr); + llvm::transform(CommandLine, CCommandLine.begin(), + [](const std::string &Str) { return Str.c_str(); }); + auto DiagOpts = CreateAndPopulateDiagOpts(CCommandLine); + sanitizeDiagOpts(*DiagOpts); + return DiagOpts; +} + class CompilerInstanceWithContext { // Context DependencyScanningWorker &Worker; @@ -236,29 +726,24 @@ class CompilerInstanceWithContext { ArrayRef CMD) : Worker(Worker), CWD(CWD), CommandLine(CMD.begin(), CMD.end()) {} - bool initialize( - DependencyActionController &Controller, - std::unique_ptr DiagEngineWithDiagOpts, - IntrusiveRefCntPtr OverlayFS) { + bool initialize(DependencyActionController &Controller, + DiagnosticsEngine &DiagEngine, + IntrusiveRefCntPtr OverlayFS) { { auto LogLine = Worker.Service.getLogger().log(); LogLine.logArray("init_compiler_instance_with_context:", " ", CommandLine); } - assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!"); ScanFS = Worker.makeEffectiveVFS(CWD, std::move(OverlayFS)); - OriginalInvocation = createCompilerInvocation( - CommandLine, *DiagEngineWithDiagOpts->DiagEngine); + OriginalInvocation = createCompilerInvocation(CommandLine, DiagEngine); if (!OriginalInvocation) { - DiagEngineWithDiagOpts->DiagEngine->Report( - diag::err_fe_expected_compiler_job) + DiagEngine.Report(diag::err_fe_expected_compiler_job) << llvm::join(CommandLine, " "); return false; } - return initializeScanInstance( - Controller, DiagEngineWithDiagOpts->DiagEngine->getClient()); + return initializeScanInstance(Controller, DiagEngine.getClient()); } bool initializeScanInstance(DependencyActionController &Controller, @@ -328,13 +813,11 @@ class CompilerInstanceWithContext { static std::optional initializeFromCC1Commandline( DependencyScanningWorker &Worker, StringRef CWD, - ArrayRef CC1CommandLine, - std::unique_ptr DiagEngineWithDiagOpts, + ArrayRef CC1CommandLine, DiagnosticsEngine &DiagEngine, IntrusiveRefCntPtr OverlayFS, DependencyActionController &Controller) { CompilerInstanceWithContext CIWC(Worker, CWD, CC1CommandLine); - if (!CIWC.initialize(Controller, std::move(DiagEngineWithDiagOpts), - std::move(OverlayFS))) + if (!CIWC.initialize(Controller, DiagEngine, std::move(OverlayFS))) return std::nullopt; return std::move(CIWC); } @@ -570,8 +1053,10 @@ bool DependencyScanningWorker::computeDependencies( Cmd); }); - auto DiagEngineWithDiagOpts = - std::make_unique(Cmd, FS, DiagConsumer); + auto DiagOpts = createScanningDiagOptions(Cmd); + auto DiagEngine = + CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer, + /*ShouldOwnClient=*/false); if (!Scanned) { // Scanning runs once for the first -cc1 invocation in a chain of driver // jobs. @@ -581,8 +1066,7 @@ bool DependencyScanningWorker::computeDependencies( // scan. Scanned = true; auto Result = CompilerInstanceWithContext::initializeFromCC1Commandline( - *this, WorkingDirectory, Cmd, std::move(DiagEngineWithDiagOpts), - OverlayFS, Controller); + *this, WorkingDirectory, Cmd, *DiagEngine, OverlayFS, Controller); if (!Result) return false; CIWC.emplace(std::move(*Result)); @@ -590,8 +1074,7 @@ bool DependencyScanningWorker::computeDependencies( return MDC != nullptr; } - auto Invocation = - createCompilerInvocation(Cmd, *DiagEngineWithDiagOpts->DiagEngine); + auto Invocation = createCompilerInvocation(Cmd, *DiagEngine); if (!Invocation) return false; @@ -617,12 +1100,14 @@ bool DependencyScanningWorker::computeDependenciesByName( llvm::function_ref()> getNextName, DependencyConsumer &DepConsumer) { auto FS = makeEffectiveVFS(CWD, OverlayFS); - auto DiagEngine = std::make_unique( - CC1CommandLine, FS, DiagConsumer); + auto DiagOpts = createScanningDiagOptions(CC1CommandLine); + auto DiagEngine = + CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer, + /*ShouldOwnClient=*/false); std::optional CIWC = CompilerInstanceWithContext::initializeFromCC1Commandline( - *this, CWD, CC1CommandLine, std::move(DiagEngine), - std::move(OverlayFS), Controller); + *this, CWD, CC1CommandLine, *DiagEngine, std::move(OverlayFS), + Controller); if (!CIWC) return false; diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp index fb3a42bc00187..b3332d9d49cd8 100644 --- a/clang/lib/Tooling/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -9,10 +9,11 @@ #include "clang/Tooling/DependencyScanningTool.h" #include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticFrontend.h" -#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/DependencyScanning/DependencyScanningWorker.h" #include "clang/Driver/Compilation.h" #include "clang/Driver/Driver.h" #include "clang/Driver/Tool.h" +#include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/Utils.h" #include "llvm/ADT/SmallVectorExtras.h" #include "llvm/ADT/iterator.h" @@ -150,10 +151,12 @@ static bool computeDependenciesForDriverCommandLine( // keep the Driver alive when we use Compilation. Arguments to commands may be // owned by Alloc when expanded from response files. llvm::BumpPtrAllocator Alloc; - auto DiagEngineWithDiagOpts = - DiagnosticsEngineWithDiagOpts(CommandLine, FS, DiagConsumer); - const auto [Driver, Compilation] = buildCompilation( - CommandLine, *DiagEngineWithDiagOpts.DiagEngine, FS, Alloc); + auto DiagOpts = createScanningDiagOptions(CommandLine); + auto DiagEngine = + CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer, + /*ShouldOwnClient=*/false); + const auto [Driver, Compilation] = + buildCompilation(CommandLine, *DiagEngine, FS, Alloc); if (!Compilation) return false; @@ -355,10 +358,12 @@ bool DependencyScanningTool::getByNameDependencies( CC1CommandLine = std::move(ModifiedCommandLine); } else { // Driver-style (or ill-formed): lower to a cc1 command line, or diagnose. - DiagnosticsEngineWithDiagOpts DiagEngineWithOpts(ModifiedCommandLine, FS, - DiagConsumer); - auto MaybeFirstCC1 = getFirstCC1CommandLine( - ModifiedCommandLine, *DiagEngineWithOpts.DiagEngine, FS); + auto DiagOpts = createScanningDiagOptions(ModifiedCommandLine); + auto DiagEngine = + CompilerInstance::createDiagnostics(*FS, *DiagOpts, &DiagConsumer, + /*ShouldOwnClient=*/false); + auto MaybeFirstCC1 = + getFirstCC1CommandLine(ModifiedCommandLine, *DiagEngine, FS); if (!MaybeFirstCC1) return false; CC1CommandLine.assign(MaybeFirstCC1->begin(), MaybeFirstCC1->end());