diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 2403cbbb652dd..18ad7bf292f1e 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -946,6 +946,12 @@ class CompilerInstance : public ModuleLoader { DependencyCollectors.push_back(std::move(Listener)); } + void clearDependencyCollectors() { DependencyCollectors.clear(); } + + std::vector> &getDependencyCollectors() { + return DependencyCollectors; + } + void setExternalSemaSource(IntrusiveRefCntPtr ESS); ModuleCache &getModuleCache() const { return *ModCache; } diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index 73308c004bd23..87a9f0d4cb06c 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -320,15 +320,6 @@ class PrintPreprocessedAction : public PreprocessorFrontendAction { bool hasPCHSupport() const override { return true; } }; -class GetDependenciesByModuleNameAction : public PreprocessOnlyAction { - StringRef ModuleName; - void ExecuteAction() override; - -public: - GetDependenciesByModuleNameAction(StringRef ModuleName) - : ModuleName(ModuleName) {} -}; - //===----------------------------------------------------------------------===// // HLSL Specific Actions //===----------------------------------------------------------------------===// diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h index 39754847a93e4..953902b13783f 100644 --- a/clang/include/clang/Lex/Preprocessor.h +++ b/clang/include/clang/Lex/Preprocessor.h @@ -1327,6 +1327,7 @@ class Preprocessor { std::move(Callbacks)); Callbacks = std::move(C); } + void removePPCallbacks() { Callbacks.reset(); } /// \} /// Get the number of tokens processed so far. diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h index f222ded8a966a..7668daf569d61 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h @@ -151,14 +151,44 @@ class DependencyScanningTool { LookupModuleOutputCallback LookupModuleOutput, std::optional TUBuffer = std::nullopt); - /// Given a compilation context specified via the Clang driver command-line, - /// gather modular dependencies of module with the given name, and return the - /// information needed for explicit build. - llvm::Expected getModuleDependencies( - StringRef ModuleName, const std::vector &CommandLine, - StringRef CWD, const llvm::DenseSet &AlreadySeen, + /// The following three methods provide a new interface to perform + /// by name dependency scan. The new interface's intention is to improve + /// dependency scanning performance when a sequence of name is looked up + /// with the same current working directory and the command line. + + /// @brief Initializing the context and the compiler instance. + /// This method must be called before calling + /// computeDependenciesByNameWithContext. + /// @param CWD The current working directory used during the scan. + /// @param CommandLine The commandline used for the scan. + /// @return Error if the initializaiton fails. + llvm::Error initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine); + + /// @brief Computes the dependeny for the module named ModuleName. + /// @param ModuleName The name of the module for which this method computes + ///. dependencies. + /// @param AlreadySeen This stores modules which have previously been + /// reported. Use the same instance for all calls to this + /// function for a single \c DependencyScanningTool in a + /// single build. Note that this parameter is not part of + /// the context because it can be shared across different + /// worker threads and each worker thread may update it. + /// @param LookupModuleOutput This function is called to fill in + /// "-fmodule-file=", "-o" and other output + /// arguments for dependencies. + /// @return An instance of \c TranslationUnitDeps if the scan is successful. + /// Otherwise it returns an error. + llvm::Expected computeDependenciesByNameWithContext( + StringRef ModuleName, const llvm::DenseSet &AlreadySeen, LookupModuleOutputCallback LookupModuleOutput); + /// @brief This method finializes the compiler instance. It finalizes the + /// diagnostics and deletes the compiler instance. Call this method + /// once all names for a same commandline are scanned. + /// @return Error if an error occured during finalization. + llvm::Error finalizeCompilerInstanceWithContext(); + llvm::vfs::FileSystem &getWorkerVFS() const { return Worker.getVFS(); } private: diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h index 6060e4b43312e..c56a2e1c87cc0 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h @@ -29,6 +29,7 @@ namespace tooling { namespace dependencies { class DependencyScanningWorkerFilesystem; +class CompilerInstanceWithContext; /// A command-line tool invocation that is part of building a TU. /// @@ -89,6 +90,8 @@ class DependencyScanningWorker { DependencyScanningWorker(DependencyScanningService &Service, llvm::IntrusiveRefCntPtr FS); + ~DependencyScanningWorker(); + /// Run the dependency scanning tool for a given clang driver command-line, /// and report the discovered dependencies to the provided consumer. If /// TUBuffer is not nullopt, it is used as TU input for the dependency @@ -103,18 +106,6 @@ class DependencyScanningWorker { DiagnosticConsumer &DiagConsumer, std::optional TUBuffer = std::nullopt); - /// Run the dependency scanning tool for a given clang driver command-line - /// for a specific module. - /// - /// \returns false if clang errors occurred (with diagnostics reported to - /// \c DiagConsumer), true otherwise. - bool computeDependencies(StringRef WorkingDirectory, - const std::vector &CommandLine, - DependencyConsumer &DepConsumer, - DependencyActionController &Controller, - DiagnosticConsumer &DiagConsumer, - StringRef ModuleName); - /// Run the dependency scanning tool for a given clang driver command-line /// for a specific translation unit via file system or memory buffer. /// @@ -125,16 +116,33 @@ class DependencyScanningWorker { DependencyConsumer &Consumer, DependencyActionController &Controller, std::optional TUBuffer = std::nullopt); - /// Run the dependency scanning tool for a given clang driver command-line - /// for a specific module. - /// - /// \returns A \c StringError with the diagnostic output if clang errors - /// occurred, success otherwise. - llvm::Error computeDependencies(StringRef WorkingDirectory, - const std::vector &CommandLine, - DependencyConsumer &Consumer, - DependencyActionController &Controller, - StringRef ModuleName); + /// The three method below implements a new interface for by name + /// dependency scanning. They together enable the dependency scanning worker + /// to more effectively perform scanning for a sequence of modules + /// by name when the CWD and CommandLine do not change across the queries. + + /// @brief Initializing the context and the compiler instance. + /// @param CWD The current working directory used during the scan. + /// @param CommandLine The commandline used for the scan. + /// @return Error if the initializaiton fails. + llvm::Error initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine); + + /// @brief Performaces dependency scanning for the module whose name is + /// specified. + /// @param ModuleName The name of the module whose dependency will be + /// scanned. + /// @param Consumer The dependency consumer that stores the results. + /// @param Controller The controller for the dependency scanning action. + /// @return Error if the scanner incurs errors. + llvm::Error + computeDependenciesByNameWithContext(StringRef ModuleName, + DependencyConsumer &Consumer, + DependencyActionController &Controller); + + /// @brief Finalizes the diagnostics engine and deletes the compiler instance. + /// @return Error if errors occur during finalization. + llvm::Error finalizeCompilerInstanceWithContext(); llvm::vfs::FileSystem &getVFS() const { return *BaseFS; } @@ -151,14 +159,16 @@ class DependencyScanningWorker { /// (passed in the constructor). llvm::IntrusiveRefCntPtr DepFS; + friend CompilerInstanceWithContext; + std::unique_ptr CIWithContext; + /// Private helper functions. bool scanDependencies(StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, DiagnosticConsumer &DC, - llvm::IntrusiveRefCntPtr FS, - std::optional ModuleName); + llvm::IntrusiveRefCntPtr FS); }; } // end namespace dependencies diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index 4136cb73f7043..9c6e3bfd8d97b 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -288,6 +288,8 @@ class ModuleDepCollector final : public DependencyCollector { void attachToPreprocessor(Preprocessor &PP) override; void attachToASTReader(ASTReader &R) override; + PPCallbacks *getPPCallbacks() { return CollectorPPPtr; } + /// Apply any changes implied by the discovered dependencies to the given /// invocation, (e.g. disable implicit modules, add explicit module paths). void applyDiscoveredDependencies(CompilerInvocation &CI); @@ -339,6 +341,10 @@ class ModuleDepCollector final : public DependencyCollector { std::optional ProvidedStdCXXModule; std::vector RequiredStdCXXModules; + /// A pointer to the preprocessor callback so we can invoke it directly + /// if needed. + ModuleDepCollectorPP *CollectorPPPtr = nullptr; + /// Checks whether the module is known as being prebuilt. bool isPrebuiltModule(const Module *M); diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index d7d56b8166350..3595bbc6c9b9e 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -1233,20 +1233,6 @@ void PrintDependencyDirectivesSourceMinimizerAction::ExecuteAction() { llvm::outs()); } -void GetDependenciesByModuleNameAction::ExecuteAction() { - CompilerInstance &CI = getCompilerInstance(); - Preprocessor &PP = CI.getPreprocessor(); - SourceManager &SM = PP.getSourceManager(); - FileID MainFileID = SM.getMainFileID(); - SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID); - SmallVector Path; - IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName); - Path.emplace_back(FileStart, ModuleID); - auto ModResult = CI.loadModule(FileStart, Path, Module::Hidden, false); - PPCallbacks *CB = PP.getPPCallbacks(); - CB->moduleImport(SourceLocation(), Path, ModResult); -} - //===----------------------------------------------------------------------===// // HLSL Specific Actions //===----------------------------------------------------------------------===// diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp index 42f52d0ff6241..cf65272b9c43d 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp @@ -1,4 +1,4 @@ -//===- DependencyScanner.cpp - Performs module dependency scanning --------===// +//===- DependencyScannerImpl.cpp - Implements module dependency scanning --===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -456,7 +456,8 @@ initVFSForTUBuferScanning(IntrusiveRefCntPtr BaseFS, return std::make_pair(ModifiedFS, ModifiedCommandLine); } -std::pair, std::vector> +std::pair, + std::vector> initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, ArrayRef CommandLine, StringRef WorkingDirectory, StringRef ModuleName) { @@ -588,7 +589,7 @@ computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, } std::unique_ptr -takeDependencyOutputOptionsFrom(CompilerInstance &ScanInstance) { +takeAndUpdateDependencyOutputOptionsFrom(CompilerInstance &ScanInstance) { // This function moves the existing dependency output options from the // invocation to the collector. The options in the invocation are reset, // which ensures that the compiler won't create new dependency collectors, @@ -675,7 +676,7 @@ bool DependencyScanningAction::runInvocation( if (!MaybePrebuiltModulesASTMap) return false; - auto DepOutputOpts = takeDependencyOutputOptionsFrom(ScanInstance); + auto DepOutputOpts = takeAndUpdateDependencyOutputOptionsFrom(ScanInstance); MDC = initializeScanInstanceDependencyCollector( ScanInstance, std::move(DepOutputOpts), WorkingDirectory, Consumer, @@ -686,8 +687,6 @@ bool DependencyScanningAction::runInvocation( if (Service.getFormat() == ScanningOutputFormat::P1689) Action = std::make_unique(); - else if (ModuleName) - Action = std::make_unique(*ModuleName); else Action = std::make_unique(); @@ -704,3 +703,163 @@ bool DependencyScanningAction::runInvocation( return Result; } + +const std::string CompilerInstanceWithContext::FakeFileBuffer = + std::string(MAX_NUM_NAMES, ' '); + +llvm::Error CompilerInstanceWithContext::initialize() { + std::tie(OverlayFS, CommandLine) = initVFSForByNameScanning( + Worker.BaseFS, CommandLine, CWD, "ScanningByName"); + + DiagPrinterWithOS = + std::make_unique(CommandLine); + DiagEngineWithCmdAndOpts = std::make_unique( + CommandLine, OverlayFS, DiagPrinterWithOS->DiagPrinter); + + std::tie(Driver, Compilation) = buildCompilation( + CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS, Alloc); + + if (!Compilation) { + return llvm::make_error("Failed to build compilation", + llvm::inconvertibleErrorCode()); + } + + assert(Compilation->getJobs().size() && + "Must have a job list of non-zero size"); + const driver::Command &Command = *(Compilation->getJobs().begin()); + const auto &CommandArgs = Command.getArguments(); + size_t ArgSize = CommandArgs.size(); + assert(ArgSize >= 1 && "Cannot have a command with 0 args"); + const char *FirstArg = CommandArgs[0]; + if (StringRef(FirstArg) != "-cc1") + return llvm::make_error( + "Incorrect compilation command, missing cc1", + llvm::inconvertibleErrorCode()); + OriginalInvocation = std::make_unique(); + + if (!CompilerInvocation::CreateFromArgs( + *OriginalInvocation, Command.getArguments(), + *DiagEngineWithCmdAndOpts->DiagEngine, Command.getExecutable())) { + DiagEngineWithCmdAndOpts->DiagEngine->Report( + diag::err_fe_expected_compiler_job) + << llvm::join(CommandLine, " "); + return llvm::make_error( + "Cannot create CompilerInvocation from Args", + llvm::inconvertibleErrorCode()); + } + + if (any(Worker.Service.getOptimizeArgs() & ScanningOptimizations::Macros)) + canonicalizeDefines(OriginalInvocation->getPreprocessorOpts()); + + // Create the CompilerInstance. + IntrusiveRefCntPtr ModCache = + makeInProcessModuleCache(Worker.Service.getModuleCacheEntries()); + CIPtr = std::make_unique( + std::make_shared(*OriginalInvocation), + Worker.PCHContainerOps, ModCache.get()); + auto &CI = *CIPtr; + + if (!initializeScanCompilerInstance( + CI, OverlayFS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(), + Worker.Service, Worker.DepFS)) { + return llvm::make_error( + "Cannot initialize scanning compiler instance", + llvm::inconvertibleErrorCode()); + } + + llvm::SmallVector StableDirs = getInitialStableDirs(CI); + auto MaybePrebuiltModulesASTMap = + computePrebuiltModulesASTMap(CI, StableDirs); + if (!MaybePrebuiltModulesASTMap) + return llvm::make_error( + "Prebuilt module scanning failed", llvm::inconvertibleErrorCode()); + + PrebuiltModuleASTMap = std::move(*MaybePrebuiltModulesASTMap); + OutputOpts = takeAndUpdateDependencyOutputOptionsFrom(CI); + + // We do not create the target in initializeScanCompilerInstance because + // setting it here is unique for by-name lookups. We create the target only + // once here, and the information is reused for all computeDependencies calls. + // We do not need to call createTarget explicitly if we go through + // CompilerInstance::ExecuteAction to perform scanning. + CI.createTarget(); + + return llvm::Error::success(); +} + +llvm::Error CompilerInstanceWithContext::computeDependencies( + StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller) { + auto &CI = *CIPtr; + + auto MDC = initializeScanInstanceDependencyCollector( + CI, std::make_unique(*OutputOpts), CWD, Consumer, + Worker.Service, + /* The MDC's constructor makes a copy of the OriginalInvocation, so + we can pass it in without worrying that it might be changed across + invocations of computeDependencies. */ + *OriginalInvocation, Controller, PrebuiltModuleASTMap, StableDirs); + + if (!SrcLocOffset) { + // When SrcLocOffset is zero, we are at the beginning of the fake source + // file. In this case, we call BeginSourceFile to initialize. + std::unique_ptr Action = + std::make_unique(); + auto InputFile = CI.getFrontendOpts().Inputs.begin(); + Action->BeginSourceFile(CI, *InputFile); + } + + Preprocessor &PP = CI.getPreprocessor(); + SourceManager &SM = PP.getSourceManager(); + FileID MainFileID = SM.getMainFileID(); + SourceLocation FileStart = SM.getLocForStartOfFile(MainFileID); + SourceLocation IDLocation = FileStart.getLocWithOffset(SrcLocOffset); + PPCallbacks *CB = nullptr; + if (!SrcLocOffset) { + // We need to call EnterSourceFile when SrcLocOffset is zero to initialize + // the preprocessor. + PP.EnterSourceFile(MainFileID, nullptr, SourceLocation()); + CB = MDC->getPPCallbacks(); + } else { + // When SrcLocOffset is non-zero, the preprocessor has already been + // initialized through a previous call of computeDependencies. We want to + // preserve the PP's state, hence we do not call EnterSourceFile again. + MDC->attachToPreprocessor(PP); + CB = MDC->getPPCallbacks(); + + FileID PrevFID; + SrcMgr::CharacteristicKind FileType = SM.getFileCharacteristic(IDLocation); + CB->LexedFileChanged(MainFileID, + PPChainedCallbacks::LexedFileChangeReason::EnterFile, + FileType, PrevFID, IDLocation); + } + + SrcLocOffset++; + SmallVector Path; + IdentifierInfo *ModuleID = PP.getIdentifierInfo(ModuleName); + Path.emplace_back(IDLocation, ModuleID); + auto ModResult = CI.loadModule(IDLocation, Path, Module::Hidden, false); + + assert(CB && "Must have PPCallbacks after module loading"); + CB->moduleImport(SourceLocation(), Path, ModResult); + // Note that we are calling the CB's EndOfMainFile function, which + // forwards the results to the dependency consumer. + // It does not indicate the end of processing the fake file. + CB->EndOfMainFile(); + + CompilerInvocation ModuleInvocation(*OriginalInvocation); + MDC->applyDiscoveredDependencies(ModuleInvocation); + Consumer.handleBuildCommand( + {CommandLine[0], ModuleInvocation.getCC1CommandLine()}); + + // Remove the DependencyCollecgtors and PPCallbacks since they are going out + // of scope. + CI.clearDependencyCollectors(); + CI.getPreprocessor().removePPCallbacks(); + return llvm::Error::success(); +} + +llvm::Error CompilerInstanceWithContext::finalize() { + DiagPrinterWithOS->DiagPrinter.finish(); + return llvm::Error::success(); +} diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h index 5657317565e8d..accd36fba6824 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h +++ b/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h @@ -1,4 +1,4 @@ -//===- DependencyScanner.h - Performs module dependency scanning *- C++ -*-===// +//===- DependencyScannerImpl.h - Implements dependency scanning *- C++ -*--===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -23,6 +23,8 @@ class DiagnosticConsumer; namespace tooling { namespace dependencies { class DependencyScanningService; +class DependencyScanningWorker; + class DependencyConsumer; class DependencyActionController; class DependencyScanningWorkerFilesystem; @@ -35,8 +37,7 @@ class DependencyScanningAction { IntrusiveRefCntPtr DepFS, std::optional ModuleName = std::nullopt) : Service(Service), WorkingDirectory(WorkingDirectory), - Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)), - ModuleName(ModuleName) {} + Consumer(Consumer), Controller(Controller), DepFS(std::move(DepFS)) {} bool runInvocation(std::unique_ptr Invocation, IntrusiveRefCntPtr FS, std::shared_ptr PCHContainerOps, @@ -66,7 +67,6 @@ class DependencyScanningAction { DependencyConsumer &Consumer; DependencyActionController &Controller; IntrusiveRefCntPtr DepFS; - std::optional ModuleName; std::optional ScanInstanceStorage; std::shared_ptr MDC; std::vector LastCC1Arguments; @@ -118,7 +118,8 @@ initVFSForTUBuferScanning(IntrusiveRefCntPtr BaseFS, StringRef WorkingDirectory, llvm::MemoryBufferRef TUBuffer); -std::pair, std::vector> +std::pair, + std::vector> initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, ArrayRef CommandLine, StringRef WorkingDirectory, StringRef ModuleName); @@ -137,7 +138,7 @@ computePrebuiltModulesASTMap(CompilerInstance &ScanInstance, SmallVector &StableDirs); std::unique_ptr -takeDependencyOutputOptionsFrom(CompilerInstance &ScanInstance); +takeAndUpdateDependencyOutputOptionsFrom(CompilerInstance &ScanInstance); /// Create the dependency collector that will collect the produced /// dependencies. May return the created ModuleDepCollector depending @@ -150,6 +151,54 @@ std::shared_ptr initializeScanInstanceDependencyCollector( DependencyActionController &Controller, PrebuiltModulesAttrsMap PrebuiltModulesASTMap, llvm::SmallVector &StableDirs); + +class CompilerInstanceWithContext { + // Context + DependencyScanningWorker &Worker; + llvm::StringRef CWD; + std::vector CommandLine; + static const uint64_t MAX_NUM_NAMES = (1 << 12); + static const std::string FakeFileBuffer; + + // Context - file systems + llvm::IntrusiveRefCntPtr OverlayFS; + + // Context - Diagnostics engine. + std::unique_ptr DiagPrinterWithOS; + std::unique_ptr DiagEngineWithCmdAndOpts; + + // Context - compiler invocation + // Compilation's command's arguments may be owned by Alloc when expanded from + // response files, so we need to keep Alloc alive in the context. + llvm::BumpPtrAllocator Alloc; + std::unique_ptr Driver; + std::unique_ptr Compilation; + std::unique_ptr OriginalInvocation; + + // Context - output options + std::unique_ptr OutputOpts; + + // Context - stable directory handling + llvm::SmallVector StableDirs; + PrebuiltModulesAttrsMap PrebuiltModuleASTMap; + + // Compiler Instance + std::unique_ptr CIPtr; + + // Source location offset. + int32_t SrcLocOffset = 0; + +public: + CompilerInstanceWithContext(DependencyScanningWorker &Worker, StringRef CWD, + const std::vector &CMD) + : Worker(Worker), CWD(CWD), CommandLine(CMD) {}; + + llvm::Error initialize(); + llvm::Error computeDependencies(StringRef ModuleName, + DependencyConsumer &Consumer, + DependencyActionController &Controller); + llvm::Error finalize(); +}; } // namespace dependencies } // namespace tooling } // namespace clang diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp index 27734ffd0e20b..f3bc99d66d07d 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp @@ -155,20 +155,29 @@ DependencyScanningTool::getTranslationUnitDependencies( return Consumer.takeTranslationUnitDeps(); } +llvm::Error DependencyScanningTool::initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine) { + return Worker.initializeCompilerInstanceWithContext(CWD, CommandLine); +} + llvm::Expected -DependencyScanningTool::getModuleDependencies( - StringRef ModuleName, const std::vector &CommandLine, - StringRef CWD, const llvm::DenseSet &AlreadySeen, +DependencyScanningTool::computeDependenciesByNameWithContext( + StringRef ModuleName, const llvm::DenseSet &AlreadySeen, LookupModuleOutputCallback LookupModuleOutput) { FullDependencyConsumer Consumer(AlreadySeen); CallbackActionController Controller(LookupModuleOutput); - llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, - Controller, ModuleName); + llvm::Error Result = Worker.computeDependenciesByNameWithContext( + ModuleName, Consumer, Controller); if (Result) return std::move(Result); + return Consumer.takeTranslationUnitDeps(); } +llvm::Error DependencyScanningTool::finalizeCompilerInstanceWithContext() { + return Worker.finalizeCompilerInstanceWithContext(); +} + TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { TranslationUnitDeps TU; diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp index 0a1cf6b18b11c..37e54a63217b3 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp @@ -43,6 +43,8 @@ DependencyScanningWorker::DependencyScanningWorker( } } +DependencyScanningWorker::~DependencyScanningWorker() = default; + llvm::Error DependencyScanningWorker::computeDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, @@ -58,21 +60,6 @@ llvm::Error DependencyScanningWorker::computeDependencies( DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); } -llvm::Error DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - StringRef ModuleName) { - // Capture the emitted diagnostics and report them to the client - // in the case of a failure. - TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine); - - if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DiagPrinterWithOS.DiagPrinter, ModuleName)) - return llvm::Error::success(); - return llvm::make_error( - DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); -} - static bool forEachDriverJob( ArrayRef ArgStrs, DiagnosticsEngine &Diags, IntrusiveRefCntPtr FS, @@ -113,11 +100,11 @@ static bool createAndRunToolInvocation( bool DependencyScanningWorker::scanDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr FS, - std::optional ModuleName) { + DiagnosticConsumer &DC, + llvm::IntrusiveRefCntPtr FS) { DignosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC); DependencyScanningAction Action(Service, WorkingDirectory, Consumer, - Controller, DepFS, ModuleName); + Controller, DepFS); bool Success = false; if (CommandLine[1] == "-cc1") { @@ -172,24 +159,32 @@ bool DependencyScanningWorker::computeDependencies( auto [FinalFS, FinalCommandLine] = initVFSForTUBuferScanning( BaseFS, CommandLine, WorkingDirectory, *TUBuffer); return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer, - Controller, DC, FinalFS, - /*ModuleName=*/std::nullopt); + Controller, DC, FinalFS); } else { BaseFS->setCurrentWorkingDirectory(WorkingDirectory); return scanDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DC, BaseFS, /*ModuleName=*/std::nullopt); + DC, BaseFS); } } -bool DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, StringRef ModuleName) { - auto [OverlayFS, ModifiedCommandLine] = initVFSForByNameScanning( - BaseFS, CommandLine, WorkingDirectory, ModuleName); +llvm::Error DependencyScanningWorker::initializeCompilerInstanceWithContext( + StringRef CWD, const std::vector &CommandLine) { + CIWithContext = + std::make_unique(*this, CWD, CommandLine); + return CIWithContext->initialize(); +} + +llvm::Error DependencyScanningWorker::computeDependenciesByNameWithContext( + StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller) { + assert(CIWithContext && "CompilerInstance with context required!"); + return CIWithContext->computeDependencies(ModuleName, Consumer, Controller); +} - return scanDependencies(WorkingDirectory, ModifiedCommandLine, Consumer, - Controller, DC, OverlayFS, ModuleName); +llvm::Error DependencyScanningWorker::finalizeCompilerInstanceWithContext() { + llvm::Error E = CIWithContext->finalize(); + CIWithContext.reset(); + return E; } DependencyActionController::~DependencyActionController() {} diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index a117bec0d656e..e07a208748b77 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -965,7 +965,9 @@ ModuleDepCollector::ModuleDepCollector( makeCommonInvocationForModuleBuild(std::move(OriginalCI))) {} void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) { - PP.addPPCallbacks(std::make_unique(*this)); + auto CollectorPP = std::make_unique(*this); + CollectorPPPtr = CollectorPP.get(); + PP.addPPCallbacks(std::move(CollectorPP)); } void ModuleDepCollector::attachToASTReader(ASTReader &R) {} diff --git a/clang/test/ClangScanDeps/link-libraries.c b/clang/test/ClangScanDeps/link-libraries.c index cc2e223102024..3719d713e775c 100644 --- a/clang/test/ClangScanDeps/link-libraries.c +++ b/clang/test/ClangScanDeps/link-libraries.c @@ -32,7 +32,7 @@ module transitive { }] // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=root > %t/result.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s // CHECK: { diff --git a/clang/test/ClangScanDeps/modules-full-by-mod-name.c b/clang/test/ClangScanDeps/modules-full-by-mod-name.c index c838614d0bfde..edb99636aaf25 100644 --- a/clang/test/ClangScanDeps/modules-full-by-mod-name.c +++ b/clang/test/ClangScanDeps/modules-full-by-mod-name.c @@ -25,7 +25,7 @@ module transitive { header "transitive.h" } }] // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=root > %t/result.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s // CHECK: { diff --git a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c new file mode 100644 index 0000000000000..030f7f3427810 --- /dev/null +++ b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c @@ -0,0 +1,108 @@ +// RUN: rm -rf %t +// RUN: split-file %s %t + +//--- module.modulemap +module root { header "root.h" } +module direct { header "direct.h" } +module transitive { header "transitive.h" } +module root1 { header "root1.h"} +//--- root.h +#include "direct.h" +#include "root/textual.h" + +//--- root1.h +#include "direct.h" + +//--- direct.h +#include "transitive.h" +//--- transitive.h +// empty + +//--- root/textual.h +// This is here to verify that the "root" directory doesn't clash with name of +// the "root" module. + +//--- cdb.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root,root1,direct > %t/result.json +// RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + +// CHECK: { +// CHECK-NEXT: "modules": [ +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "transitive" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/direct.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "direct" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "direct" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/root.h", +// CHECK-NEXT: "[[PREFIX]]/root/textual.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "root" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [ +// CHECK-NEXT: { +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "module-name": "direct" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/root1.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "root1" +// CHECK-NEXT: }, +// CHECK-NEXT: { +// CHECK-NEXT: "clang-module-deps": [], +// CHECK-NEXT: "clang-modulemap-file": "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "command-line": [ +// CHECK: ], +// CHECK-NEXT: "context-hash": "{{.*}}", +// CHECK-NEXT: "file-deps": [ +// CHECK-NEXT: "[[PREFIX]]/module.modulemap", +// CHECK-NEXT: "[[PREFIX]]/transitive.h" +// CHECK-NEXT: ], +// CHECK-NEXT: "link-libraries": [], +// CHECK-NEXT: "name": "transitive" +// CHECK-NEXT: } +// CHECK-NEXT: ], +// CHECK-NEXT: "translation-units": [] +// CHECK-NEXT: } diff --git a/clang/test/Modules/transitive-system.test b/clang/test/Modules/transitive-system.test index b1f1558b31742..5f6196cc1d6a3 100644 --- a/clang/test/Modules/transitive-system.test +++ b/clang/test/Modules/transitive-system.test @@ -2,9 +2,9 @@ // RUN: split-file %s %t // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=direct > %t/result1.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=direct > %t/result1.json // RUN: rm -rf %t/cache -// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-name=transitive > %t/result2.json +// RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=transitive > %t/result2.json // RUN: %deps-to-rsp %t/result1.json --module-name transitive > %t/1.rsp // RUN: %deps-to-rsp %t/result2.json --module-name transitive > %t/2.rsp // RUN: diff %t/1.rsp %t/2.rsp diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index c11a34870b204..65f0f40cc5b75 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -87,7 +87,7 @@ static std::string ModuleFilesDir; static bool EagerLoadModules; static unsigned NumThreads = 0; static std::string CompilationDB; -static std::optional ModuleName; +static std::optional ModuleNames; static std::vector ModuleDepTargets; static std::string TranslationUnitFile; static bool DeprecatedDriverCommand; @@ -205,8 +205,8 @@ static void ParseArgs(int argc, char **argv) { if (const llvm::opt::Arg *A = Args.getLastArg(OPT_compilation_database_EQ)) CompilationDB = A->getValue(); - if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_name_EQ)) - ModuleName = A->getValue(); + if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_names_EQ)) + ModuleNames = A->getValue(); for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ)) ModuleDepTargets.emplace_back(A->getValue()); @@ -664,6 +664,16 @@ static bool handleModuleResult(StringRef ModuleName, return false; } +static void handleErrorWithInfoString(StringRef Info, llvm::Error E, + SharedStream &OS, SharedStream &Errs) { + llvm::handleAllErrors(std::move(E), [&Info, &Errs](llvm::StringError &Err) { + Errs.applyLocked([&](raw_ostream &OS) { + OS << "Error: " << Info << ":\n"; + OS << Err.getMessage(); + }); + }); +} + class P1689Deps { public: void printDependencies(raw_ostream &OS) { @@ -1008,7 +1018,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { }; if (Format == ScanningOutputFormat::Full) - FD.emplace(!ModuleName ? Inputs.size() : 0); + FD.emplace(!ModuleNames ? Inputs.size() : 0); std::atomic NumStatusCalls = 0; std::atomic NumOpenFileForReadCalls = 0; @@ -1082,13 +1092,38 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { MakeformatOS, Errs)) HadErrors = true; } - } else if (ModuleName) { - auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies( - *ModuleName, Input->CommandLine, CWD, AlreadySeenModules, - LookupOutput); - if (handleModuleResult(*ModuleName, MaybeModuleDepsGraph, *FD, - LocalIndex, DependencyOS, Errs)) + } else if (ModuleNames) { + StringRef ModuleNameRef(*ModuleNames); + SmallVector Names; + ModuleNameRef.split(Names, ','); + + if (llvm::Error Err = WorkerTool.initializeCompilerInstanceWithContext( + CWD, Input->CommandLine)) { + handleErrorWithInfoString( + "Compiler instance with context setup error", std::move(Err), + DependencyOS, Errs); HadErrors = true; + continue; + } + + for (auto N : Names) { + auto MaybeModuleDepsGraph = + WorkerTool.computeDependenciesByNameWithContext( + N, AlreadySeenModules, LookupOutput); + if (handleModuleResult(N, MaybeModuleDepsGraph, *FD, LocalIndex, + DependencyOS, Errs)) { + HadErrors = true; + break; + } + } + + if (llvm::Error Err = + WorkerTool.finalizeCompilerInstanceWithContext()) { + handleErrorWithInfoString( + "Compiler instance with context finialization error", + std::move(Err), DependencyOS, Errs); + HadErrors = true; + } } else { std::unique_ptr TU; std::optional TUBuffer; diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td index 7a63b18f6d462..6ea9d824c9646 100644 --- a/clang/tools/clang-scan-deps/Opts.td +++ b/clang/tools/clang-scan-deps/Opts.td @@ -26,7 +26,9 @@ def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of laz def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">; defm compilation_database : Eq<"compilation-database", "Compilation database">; -defm module_name : Eq<"module-name", "the module of which the dependencies are to be computed">; +defm module_names + : Eq<"module-names", "A comma separated list of names of modules of which " + "the dependencies are to be computed">; defm dependency_target : Eq<"dependency-target", "The names of dependency targets for the dependency file">; defm tu_buffer_path: Eq<"tu-buffer-path", "The path to the translation unit for depscan. Not compatible with -module-name">;