diff --git a/clang-tools-extra/clangd/ScanningProjectModules.cpp b/clang-tools-extra/clangd/ScanningProjectModules.cpp index 672e99632019d..6a21ad2920764 100644 --- a/clang-tools-extra/clangd/ScanningProjectModules.cpp +++ b/clang-tools-extra/clangd/ScanningProjectModules.cpp @@ -8,8 +8,8 @@ #include "ProjectModules.h" #include "support/Logger.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "clang/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanningTool.h" namespace clang::clangd { namespace { @@ -36,8 +36,8 @@ class ModuleDependencyScanner { std::shared_ptr CDB, const ThreadsafeFS &TFS) : CDB(CDB), TFS(TFS), - Service(tooling::dependencies::ScanningMode::CanonicalPreprocessing, - tooling::dependencies::ScanningOutputFormat::P1689) {} + Service(dependencies::ScanningMode::CanonicalPreprocessing, + dependencies::ScanningOutputFormat::P1689) {} /// The scanned modules dependency information for a specific source file. struct ModuleDependencyInfo { @@ -81,7 +81,7 @@ class ModuleDependencyScanner { // Whether the scanner has scanned the project globally. bool GlobalScanned = false; - clang::tooling::dependencies::DependencyScanningService Service; + clang::dependencies::DependencyScanningService Service; // TODO: Add a scanning cache. diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h similarity index 76% rename from clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h rename to clang/include/clang/DependencyScanning/DependencyScannerImpl.h index b94d1b472f920..61d0aa59a3fb1 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.h +++ b/clang/include/clang/DependencyScanning/DependencyScannerImpl.h @@ -9,18 +9,17 @@ #ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H #define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNER_H +#include "clang/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/TextDiagnosticPrinter.h" -#include "clang/Serialization/ObjectFilePCHContainerReader.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" -#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" namespace clang { class DiagnosticConsumer; -namespace tooling { namespace dependencies { class DependencyScanningService; class DependencyScanningWorker; @@ -103,27 +102,10 @@ struct TextDiagnosticsPrinterWithOutput { DiagPrinter(DiagnosticsOS, *DiagOpts) {} }; -std::pair, std::unique_ptr> -buildCompilation(ArrayRef ArgStrs, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr FS, - llvm::BumpPtrAllocator &Alloc); - std::unique_ptr createCompilerInvocation(ArrayRef CommandLine, DiagnosticsEngine &Diags); -std::pair, std::vector> -initVFSForTUBufferScanning(IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, - StringRef WorkingDirectory, - llvm::MemoryBufferRef TUBuffer); - -std::pair, - std::vector> -initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, - StringRef WorkingDirectory, StringRef ModuleName); - bool initializeScanCompilerInstance( CompilerInstance &ScanInstance, IntrusiveRefCntPtr FS, @@ -158,22 +140,11 @@ class CompilerInstanceWithContext { llvm::StringRef CWD; std::vector CommandLine; - // Context - file systems - llvm::IntrusiveRefCntPtr OverlayFS; - // Context - Diagnostics engine. - std::unique_ptr DiagPrinterWithOS; - // DiagConsumer may points to DiagPrinterWithOS->DiagPrinter, or a custom - // DiagnosticConsumer passed in from initialize. DiagnosticConsumer *DiagConsumer = nullptr; std::unique_ptr DiagEngineWithCmdAndOpts; // Context - compiler invocation - // Compilation's command's arguments may be owned by Alloc when expanded from - // response files, so we need to keep Alloc alive in the context. - llvm::BumpPtrAllocator Alloc; - std::unique_ptr Driver; - std::unique_ptr Compilation; std::unique_ptr OriginalInvocation; // Context - output options @@ -195,18 +166,15 @@ class CompilerInstanceWithContext { : Worker(Worker), CWD(CWD), CommandLine(CMD) {}; // The three methods below returns false when they fail, with the detail - // accumulated in DiagConsumer. - bool initialize(DiagnosticConsumer *DC); + // accumulated in \c DiagEngineWithDiagOpts's diagnostic consumer. + bool initialize( + std::unique_ptr DiagEngineWithDiagOpts, + IntrusiveRefCntPtr FS); bool computeDependencies(StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller); bool finalize(); - - // The method below turns the return status from the above methods - // into an llvm::Error using a default DiagnosticConsumer. - llvm::Error handleReturnStatus(bool Success); }; } // namespace dependencies -} // namespace tooling } // namespace clang #endif diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/DependencyScanning/DependencyScanningFilesystem.h similarity index 98% rename from clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h rename to clang/include/clang/DependencyScanning/DependencyScanningFilesystem.h index 2b21be7712693..a4516ff77509d 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningFilesystem.h @@ -1,4 +1,4 @@ -//===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===// +//===- DependencyScanningFilesystem.h - Optimized Scanning FS ---*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H -#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H +#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H +#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H #include "clang/Basic/LLVM.h" #include "clang/Lex/DependencyDirectivesScanner.h" @@ -21,7 +21,6 @@ #include namespace clang { -namespace tooling { namespace dependencies { using DependencyDirectivesTy = @@ -521,7 +520,6 @@ class DependencyScanningWorkerFilesystem }; } // end namespace dependencies -} // end namespace tooling } // end namespace clang -#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H +#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGFILESYSTEM_H diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/DependencyScanning/DependencyScanningService.h similarity index 89% rename from clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h rename to clang/include/clang/DependencyScanning/DependencyScanningService.h index 4e97c7bc9f36e..96dd33c28cf5a 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningService.h @@ -1,4 +1,4 @@ -//===- DependencyScanningService.h - clang-scan-deps service ===-*- C++ -*-===// +//===- DependencyScanningService.h - Scanning Service -----------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,16 +6,15 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H -#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H +#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H +#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H -#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" -#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h" +#include "clang/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/DependencyScanning/InProcessModuleCache.h" #include "llvm/ADT/BitmaskEnum.h" #include "llvm/Support/Chrono.h" namespace clang { -namespace tooling { namespace dependencies { /// The mode in which the dependency scanner will operate to find the @@ -125,7 +124,6 @@ class DependencyScanningService { }; } // end namespace dependencies -} // end namespace tooling } // end namespace clang -#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H +#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGSERVICE_H diff --git a/clang/include/clang/DependencyScanning/DependencyScanningUtils.h b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h new file mode 100644 index 0000000000000..e2fb5ad3e5cf3 --- /dev/null +++ b/clang/include/clang/DependencyScanning/DependencyScanningUtils.h @@ -0,0 +1,166 @@ +//===- DependencyScanningUtils.h - Common Scanning Utilities ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGUTILS_H +#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGUTILS_H + +#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/DependencyScanning/DependencyScanningWorker.h" +#include "clang/DependencyScanning/ModuleDepCollector.h" + +namespace clang { +namespace dependencies { + +/// Graph of modular dependencies. +using ModuleDepsGraph = std::vector; + +/// The full dependencies and module graph for a specific input. +struct TranslationUnitDeps { + /// The graph of direct and transitive modular dependencies. + ModuleDepsGraph ModuleGraph; + + /// The identifier of the C++20 module this translation unit exports. + /// + /// If the translation unit is not a module then \c ID.ModuleName is empty. + clang::dependencies::ModuleID ID; + + /// A collection of absolute paths to files that this translation unit + /// directly depends on, not including transitive dependencies. + std::vector FileDeps; + + /// A collection of prebuilt modules this translation unit directly depends + /// on, not including transitive dependencies. + std::vector PrebuiltModuleDeps; + + /// A list of modules this translation unit directly depends on, not including + /// transitive dependencies. + /// + /// This may include modules with a different context hash when it can be + /// determined that the differences are benign for this compilation. + std::vector ClangModuleDeps; + + /// A list of module names that are visible to this translation unit. This + /// includes both direct and transitive module dependencies. + std::vector VisibleModules; + + /// A list of the C++20 named modules this translation unit depends on. + std::vector NamedModuleDeps; + + /// The sequence of commands required to build the translation unit. Commands + /// should be executed in order. + /// + /// FIXME: If we add support for multi-arch builds in clang-scan-deps, we + /// should make the dependencies between commands explicit to enable parallel + /// builds of each architecture. + std::vector Commands; + + /// Deprecated driver command-line. This will be removed in a future version. + std::vector DriverCommandLine; +}; + +class FullDependencyConsumer : public clang::dependencies::DependencyConsumer { +public: + FullDependencyConsumer( + const llvm::DenseSet &AlreadySeen) + : AlreadySeen(AlreadySeen) {} + + void handleBuildCommand(clang::dependencies::Command Cmd) override { + Commands.push_back(std::move(Cmd)); + } + + void handleDependencyOutputOpts(const DependencyOutputOptions &) override {} + + void handleFileDependency(StringRef File) override { + Dependencies.push_back(std::string(File)); + } + + void handlePrebuiltModuleDependency( + clang::dependencies::PrebuiltModuleDep PMD) override { + PrebuiltModuleDeps.emplace_back(std::move(PMD)); + } + + void handleModuleDependency(clang::dependencies::ModuleDeps MD) override { + ClangModuleDeps[MD.ID] = std::move(MD); + } + + void handleDirectModuleDependency(clang::dependencies::ModuleID ID) override { + DirectModuleDeps.push_back(ID); + } + + void handleVisibleModule(std::string ModuleName) override { + VisibleModules.push_back(ModuleName); + } + + void handleContextHash(std::string Hash) override { + ContextHash = std::move(Hash); + } + + void handleProvidedAndRequiredStdCXXModules( + std::optional Provided, + std::vector Requires) override { + ModuleName = Provided ? Provided->ModuleName : ""; + llvm::transform(Requires, std::back_inserter(NamedModuleDeps), + [](const auto &Module) { return Module.ModuleName; }); + } + + TranslationUnitDeps takeTranslationUnitDeps(); + +private: + std::vector Dependencies; + std::vector PrebuiltModuleDeps; + llvm::MapVector + ClangModuleDeps; + std::string ModuleName; + std::vector NamedModuleDeps; + std::vector DirectModuleDeps; + std::vector VisibleModules; + std::vector Commands; + std::string ContextHash; + const llvm::DenseSet &AlreadySeen; +}; + +/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc. +using LookupModuleOutputCallback = + llvm::function_ref; + +/// A simple dependency action controller that uses a callback. If no callback +/// is provided, it is assumed that looking up module outputs is unreachable. +class CallbackActionController + : public clang::dependencies::DependencyActionController { +public: + virtual ~CallbackActionController(); + + static std::string + lookupUnreachableModuleOutput(const clang::dependencies::ModuleDeps &MD, + clang::dependencies::ModuleOutputKind Kind) { + llvm::report_fatal_error("unexpected call to lookupModuleOutput"); + }; + + CallbackActionController(LookupModuleOutputCallback LMO) + : LookupModuleOutput(std::move(LMO)) { + if (!LookupModuleOutput) { + LookupModuleOutput = lookupUnreachableModuleOutput; + } + } + + std::string + lookupModuleOutput(const clang::dependencies::ModuleDeps &MD, + clang::dependencies::ModuleOutputKind Kind) override { + return LookupModuleOutput(MD, Kind); + } + +private: + LookupModuleOutputCallback LookupModuleOutput; +}; + +} // end namespace dependencies +} // end namespace clang + +#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGUTILS_H diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h similarity index 69% rename from clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h rename to clang/include/clang/DependencyScanning/DependencyScanningWorker.h index e2c353a254bf3..0f1b78648342f 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h +++ b/clang/include/clang/DependencyScanning/DependencyScanningWorker.h @@ -1,4 +1,4 @@ -//===- DependencyScanningWorker.h - clang-scan-deps worker ===---*- C++ -*-===// +//===- DependencyScanningWorker.h - Thread-Safe Scanning Worker -*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,15 +6,17 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H -#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H +#ifndef LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H +#define LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H +#include "clang/Basic/Diagnostic.h" #include "clang/Basic/DiagnosticOptions.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/LLVM.h" +#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/DependencyScanning/DependencyScanningService.h" +#include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Frontend/PCHContainerOperations.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" -#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" #include "llvm/Support/Error.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/MemoryBufferRef.h" @@ -25,7 +27,6 @@ namespace clang { class DependencyOutputOptions; -namespace tooling { namespace dependencies { class DependencyScanningWorkerFilesystem; @@ -92,41 +93,56 @@ class DependencyScanningWorker { ~DependencyScanningWorker(); - /// Run the dependency scanning tool for a given clang driver command-line, - /// and report the discovered dependencies to the provided consumer. If - /// TUBuffer is not nullopt, it is used as TU input for the dependency - /// scanning. Otherwise, the input should be included as part of the - /// command-line. + /// Run the dependency scanning tool for a given clang -cc1 command-line, + /// and report the discovered dependencies to the provided consumer. /// - /// \returns false if clang errors occurred (with diagnostics reported to + /// @return false if clang errors occurred (with diagnostics reported to /// \c DiagConsumer), true otherwise. bool computeDependencies( StringRef WorkingDirectory, const std::vector &CommandLine, DependencyConsumer &DepConsumer, DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer, - std::optional TUBuffer = std::nullopt); + llvm::IntrusiveRefCntPtr ScanFS = nullptr); - /// Run the dependency scanning tool for a given clang driver command-line - /// for a specific translation unit via file system or memory buffer. + /// Run the dependency scanning tool for all given clang -cc1 command-lines, + /// and report the discovered dependencies to the provided consumer. /// - /// \returns A \c StringError with the diagnostic output if clang errors - /// occurred, success otherwise. - llvm::Error computeDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, + /// \returns false if clang errors occurred (with diagnostics reported to + /// \c Diags), true otherwise. + bool computeDependencies( + StringRef WorkingDirectory, + ArrayRef> CommandLines, DependencyConsumer &Consumer, DependencyActionController &Controller, - std::optional TUBuffer = std::nullopt); + DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr ScanFS = nullptr); /// The three method below implements a new interface for by name /// dependency scanning. They together enable the dependency scanning worker /// to more effectively perform scanning for a sequence of modules /// by name when the CWD and CommandLine do not change across the queries. + /// The initialization function asks the client for a DiagnosticsConsumer + /// that it direct the diagnostics to. /// @brief Initializing the context and the compiler instance. /// @param CWD The current working directory used during the scan. /// @param CommandLine The commandline used for the scan. - /// @return Error if the initializaiton fails. - llvm::Error initializeCompilerInstanceWithContextOrError( - StringRef CWD, const std::vector &CommandLine); + /// @return False if the initializaiton fails. + bool initializeCompilerInstanceWithContext(StringRef CWD, + ArrayRef CommandLine, + DiagnosticConsumer &DC); + + /// @brief Initializing the context and the compiler instance. + /// @param CWD The current working directory used during the scan. + /// @param CommandLine The commandline used for the scan. + /// @param DiagEngineWithCmdAndOpts Preconfigured diagnostics engine and + /// options associated with the cc1 command line. + /// @param FS The file system (typically an overlay) to use for this compiler + /// instance. + /// @return False if the initializaiton fails. + bool initializeCompilerInstanceWithContext( + StringRef CWD, ArrayRef CommandLine, + std::unique_ptr DiagEngineWithCmdAndOpts, + IntrusiveRefCntPtr FS); /// @brief Performaces dependency scanning for the module whose name is /// specified. @@ -134,28 +150,15 @@ class DependencyScanningWorker { /// scanned. /// @param Consumer The dependency consumer that stores the results. /// @param Controller The controller for the dependency scanning action. - /// @return Error if the scanner incurs errors. - llvm::Error computeDependenciesByNameWithContextOrError( - StringRef ModuleName, DependencyConsumer &Consumer, - DependencyActionController &Controller); - - /// @brief Finalizes the diagnostics engine and deletes the compiler instance. - /// @return Error if errors occur during finalization. - llvm::Error finalizeCompilerInstanceWithContextOrError(); - - /// The three methods below provides the same functionality as the - /// three methods above. Instead of returning `llvm::Error`s, these - /// three methods return a flag to indicate if the call is successful. - /// The initialization function asks the client for a DiagnosticsConsumer - /// that it direct the diagnostics to. - bool initializeCompilerInstanceWithContext( - StringRef CWD, const std::vector &CommandLine, - DiagnosticConsumer *DC = nullptr); + /// @return False if the scanner incurs errors. bool computeDependenciesByNameWithContext(StringRef ModuleName, DependencyConsumer &Consumer, DependencyActionController &Controller); - bool finalizeCompilerInstance(); + + /// @brief Finalizes the diagnostics engine and deletes the compiler instance. + /// @return False if errors occur during finalization. + bool finalizeCompilerInstanceWithContext(); llvm::vfs::FileSystem &getVFS() const { return *BaseFS; } @@ -182,10 +185,16 @@ class DependencyScanningWorker { DependencyActionController &Controller, DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr FS); + + bool scanDependencies(StringRef WorkingDirectory, + ArrayRef> CommandLine, + DependencyConsumer &Consumer, + DependencyActionController &Controller, + DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr FS); }; } // end namespace dependencies -} // end namespace tooling } // end namespace clang -#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H +#endif // LLVM_CLANG_DEPENDENCYSCANNING_DEPENDENCYSCANNINGWORKER_H diff --git a/clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h b/clang/include/clang/DependencyScanning/InProcessModuleCache.h similarity index 75% rename from clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h rename to clang/include/clang/DependencyScanning/InProcessModuleCache.h index 213e60b39c199..c0e8f00b7fb59 100644 --- a/clang/include/clang/Tooling/DependencyScanning/InProcessModuleCache.h +++ b/clang/include/clang/DependencyScanning/InProcessModuleCache.h @@ -1,4 +1,4 @@ -//===----------------------------------------------------------------------===// +//===- InProcessModuleCache.h - Implicit Module Cache -----------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,8 +6,8 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H -#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H +#ifndef LLVM_CLANG_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H +#define LLVM_CLANG_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H #include "clang/Serialization/ModuleCache.h" #include "llvm/ADT/StringMap.h" @@ -16,8 +16,8 @@ #include namespace clang { -namespace tooling { namespace dependencies { + struct ModuleCacheEntry { std::shared_mutex CompilationMutex; std::atomic Timestamp = 0; @@ -30,8 +30,8 @@ struct ModuleCacheEntries { IntrusiveRefCntPtr makeInProcessModuleCache(ModuleCacheEntries &Entries); + } // namespace dependencies -} // namespace tooling } // namespace clang -#endif +#endif // LLVM_CLANG_DEPENDENCYSCANNING_INPROCESSMODULECACHE_H diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/DependencyScanning/ModuleDepCollector.h similarity index 95% rename from clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h rename to clang/include/clang/DependencyScanning/ModuleDepCollector.h index b0a91b60ff6da..0243f7abcbe10 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/DependencyScanning/ModuleDepCollector.h @@ -6,18 +6,18 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H -#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H +#ifndef LLVM_CLANG_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H +#define LLVM_CLANG_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H #include "clang/Basic/LLVM.h" #include "clang/Basic/Module.h" #include "clang/Basic/SourceManager.h" +#include "clang/DependencyScanning/DependencyScanningService.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/Utils.h" #include "clang/Lex/HeaderSearch.h" #include "clang/Lex/PPCallbacks.h" #include "clang/Serialization/ASTReader.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringSet.h" @@ -28,7 +28,6 @@ #include namespace clang { -namespace tooling { namespace dependencies { class DependencyActionController; @@ -109,7 +108,7 @@ struct ModuleID { std::tie(Other.ModuleName, Other.ContextHash); } - bool operator<(const ModuleID& Other) const { + bool operator<(const ModuleID &Other) const { return std::tie(ModuleName, ContextHash) < std::tie(Other.ModuleName, Other.ContextHash); } @@ -264,10 +263,11 @@ class ModuleDepCollectorPP final : public PPCallbacks { /// Traverses the affecting modules and updates \c MD with references to the /// parent \c ModuleDepCollector info. - void addAllAffectingClangModules(const Module *M, ModuleDeps &MD, + void + addAllAffectingClangModules(const Module *M, ModuleDeps &MD, llvm::DenseSet &AddedModules); void addAffectingClangModule(const Module *M, ModuleDeps &MD, - llvm::DenseSet &AddedModules); + llvm::DenseSet &AddedModules); /// Add discovered module dependency for the given module. void addOneModuleDep(const Module *M, const ModuleID ID, ModuleDeps &MD); @@ -406,16 +406,15 @@ bool areOptionsInStableDir(const ArrayRef Directories, const HeaderSearchOptions &HSOpts); } // end namespace dependencies -} // end namespace tooling } // end namespace clang namespace llvm { -inline hash_code hash_value(const clang::tooling::dependencies::ModuleID &ID) { +inline hash_code hash_value(const clang::dependencies::ModuleID &ID) { return hash_combine(ID.ModuleName, ID.ContextHash); } -template <> struct DenseMapInfo { - using ModuleID = clang::tooling::dependencies::ModuleID; +template <> struct DenseMapInfo { + using ModuleID = clang::dependencies::ModuleID; static inline ModuleID getEmptyKey() { return ModuleID{"", ""}; } static inline ModuleID getTombstoneKey() { return ModuleID{"~", "~"}; // ~ is not a valid module name or context hash @@ -427,4 +426,4 @@ template <> struct DenseMapInfo { }; } // namespace llvm -#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H +#endif // LLVM_CLANG_DEPENDENCYSCANNING_MODULEDEPCOLLECTOR_H diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h b/clang/include/clang/Tooling/DependencyScanningTool.h similarity index 50% rename from clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h rename to clang/include/clang/Tooling/DependencyScanningTool.h index ed562f46cfdaa..6073d9a567ede 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h +++ b/clang/include/clang/Tooling/DependencyScanningTool.h @@ -6,12 +6,14 @@ // //===----------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGTOOL_H -#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGTOOL_H - -#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" -#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" +#ifndef LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H +#define LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H + +#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/DependencyScanning/DependencyScanningService.h" +#include "clang/DependencyScanning/DependencyScanningUtils.h" +#include "clang/DependencyScanning/DependencyScanningWorker.h" +#include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Tooling/JSONCompilationDatabase.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/MapVector.h" @@ -25,61 +27,10 @@ namespace clang { namespace tooling { namespace dependencies { -/// A callback to lookup module outputs for "-fmodule-file=", "-o" etc. -using LookupModuleOutputCallback = - llvm::function_ref; - -/// Graph of modular dependencies. -using ModuleDepsGraph = std::vector; - -/// The full dependencies and module graph for a specific input. -struct TranslationUnitDeps { - /// The graph of direct and transitive modular dependencies. - ModuleDepsGraph ModuleGraph; - - /// The identifier of the C++20 module this translation unit exports. - /// - /// If the translation unit is not a module then \c ID.ModuleName is empty. - ModuleID ID; - - /// A collection of absolute paths to files that this translation unit - /// directly depends on, not including transitive dependencies. - std::vector FileDeps; - - /// A collection of prebuilt modules this translation unit directly depends - /// on, not including transitive dependencies. - std::vector PrebuiltModuleDeps; - - /// A list of modules this translation unit directly depends on, not including - /// transitive dependencies. - /// - /// This may include modules with a different context hash when it can be - /// determined that the differences are benign for this compilation. - std::vector ClangModuleDeps; - - /// A list of module names that are visible to this translation unit. This - /// includes both direct and transitive module dependencies. - std::vector VisibleModules; - - /// A list of the C++20 named modules this translation unit depends on. - std::vector NamedModuleDeps; - - /// The sequence of commands required to build the translation unit. Commands - /// should be executed in order. - /// - /// FIXME: If we add support for multi-arch builds in clang-scan-deps, we - /// should make the dependencies between commands explicit to enable parallel - /// builds of each architecture. - std::vector Commands; - - /// Deprecated driver command-line. This will be removed in a future version. - std::vector DriverCommandLine; -}; - struct P1689Rule { std::string PrimaryOutput; - std::optional Provides; - std::vector Requires; + std::optional Provides; + std::vector Requires; }; /// The high-level implementation of the dependency discovery tool that runs on @@ -90,9 +41,10 @@ class DependencyScanningTool { /// /// @param Service The parent service. Must outlive the tool. /// @param FS The filesystem for the tool to use. Defaults to the physical FS. - DependencyScanningTool(DependencyScanningService &Service, - llvm::IntrusiveRefCntPtr FS = - llvm::vfs::createPhysicalFileSystem()); + DependencyScanningTool( + clang::dependencies::DependencyScanningService &Service, + llvm::IntrusiveRefCntPtr FS = + llvm::vfs::createPhysicalFileSystem()); /// Print out the dependency information into a string using the dependency /// file format that is specified in the options (-MD is the default) and @@ -145,10 +97,11 @@ class DependencyScanningTool { /// /// \returns a \c StringError with the diagnostic output if clang errors /// occurred, \c TranslationUnitDeps otherwise. - llvm::Expected getTranslationUnitDependencies( + llvm::Expected + getTranslationUnitDependencies( const std::vector &CommandLine, StringRef CWD, - const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, + const llvm::DenseSet &AlreadySeen, + clang::dependencies::LookupModuleOutputCallback LookupModuleOutput, std::optional TUBuffer = std::nullopt); /// Given a compilation context specified via the Clang driver command-line, @@ -157,10 +110,12 @@ class DependencyScanningTool { /// TODO: this method should be removed as soon as Swift and our C-APIs adopt /// CompilerInstanceWithContext. We are keeping it here so that it is easier /// to coordinate with Swift and C-API changes. - llvm::Expected getModuleDependencies( + llvm::Expected + getModuleDependencies( StringRef ModuleName, const std::vector &CommandLine, - StringRef CWD, const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput); + StringRef CWD, + const llvm::DenseSet &AlreadySeen, + clang::dependencies::LookupModuleOutputCallback LookupModuleOutput); /// The following three methods provide a new interface to perform /// by name dependency scan. The new interface's intention is to improve @@ -173,8 +128,8 @@ class DependencyScanningTool { /// @param CWD The current working directory used during the scan. /// @param CommandLine The commandline used for the scan. /// @return Error if the initializaiton fails. - llvm::Error initializeCompilerInstanceWithContext( - StringRef CWD, const std::vector &CommandLine); + llvm::Error initializeCompilerInstanceWithContextOrError( + StringRef CWD, ArrayRef CommandLine); /// @brief Computes the dependeny for the module named ModuleName. /// @param ModuleName The name of the module for which this method computes @@ -190,109 +145,28 @@ class DependencyScanningTool { /// arguments for dependencies. /// @return An instance of \c TranslationUnitDeps if the scan is successful. /// Otherwise it returns an error. - llvm::Expected computeDependenciesByNameWithContext( - StringRef ModuleName, const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput); + llvm::Expected + computeDependenciesByNameWithContextOrError( + StringRef ModuleName, + const llvm::DenseSet &AlreadySeen, + clang::dependencies::LookupModuleOutputCallback LookupModuleOutput); /// @brief This method finializes the compiler instance. It finalizes the /// diagnostics and deletes the compiler instance. Call this method /// once all names for a same commandline are scanned. /// @return Error if an error occured during finalization. - llvm::Error finalizeCompilerInstanceWithContext(); + llvm::Error finalizeCompilerInstanceWithContextOrError(); llvm::vfs::FileSystem &getWorkerVFS() const { return Worker.getVFS(); } private: - DependencyScanningWorker Worker; -}; - -class FullDependencyConsumer : public DependencyConsumer { -public: - FullDependencyConsumer(const llvm::DenseSet &AlreadySeen) - : AlreadySeen(AlreadySeen) {} - - void handleBuildCommand(Command Cmd) override { - Commands.push_back(std::move(Cmd)); - } - - void handleDependencyOutputOpts(const DependencyOutputOptions &) override {} - - void handleFileDependency(StringRef File) override { - Dependencies.push_back(std::string(File)); - } - - void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override { - PrebuiltModuleDeps.emplace_back(std::move(PMD)); - } - - void handleModuleDependency(ModuleDeps MD) override { - ClangModuleDeps[MD.ID] = std::move(MD); - } - - void handleDirectModuleDependency(ModuleID ID) override { - DirectModuleDeps.push_back(ID); - } - - void handleVisibleModule(std::string ModuleName) override { - VisibleModules.push_back(ModuleName); - } - - void handleContextHash(std::string Hash) override { - ContextHash = std::move(Hash); - } - - void handleProvidedAndRequiredStdCXXModules( - std::optional Provided, - std::vector Requires) override { - ModuleName = Provided ? Provided->ModuleName : ""; - llvm::transform(Requires, std::back_inserter(NamedModuleDeps), - [](const auto &Module) { return Module.ModuleName; }); - } - - TranslationUnitDeps takeTranslationUnitDeps(); - -private: - std::vector Dependencies; - std::vector PrebuiltModuleDeps; - llvm::MapVector ClangModuleDeps; - std::string ModuleName; - std::vector NamedModuleDeps; - std::vector DirectModuleDeps; - std::vector VisibleModules; - std::vector Commands; - std::string ContextHash; - const llvm::DenseSet &AlreadySeen; -}; - -/// A simple dependency action controller that uses a callback. If no callback -/// is provided, it is assumed that looking up module outputs is unreachable. -class CallbackActionController : public DependencyActionController { -public: - virtual ~CallbackActionController(); - - static std::string lookupUnreachableModuleOutput(const ModuleDeps &MD, - ModuleOutputKind Kind) { - llvm::report_fatal_error("unexpected call to lookupModuleOutput"); - }; - - CallbackActionController(LookupModuleOutputCallback LMO) - : LookupModuleOutput(std::move(LMO)) { - if (!LookupModuleOutput) { - LookupModuleOutput = lookupUnreachableModuleOutput; - } - } - - std::string lookupModuleOutput(const ModuleDeps &MD, - ModuleOutputKind Kind) override { - return LookupModuleOutput(MD, Kind); - } - -private: - LookupModuleOutputCallback LookupModuleOutput; + clang::dependencies::DependencyScanningWorker Worker; + std::unique_ptr + DiagPrinterWithOS; }; } // end namespace dependencies } // end namespace tooling } // end namespace clang -#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNING_DEPENDENCYSCANNINGTOOL_H +#endif // LLVM_CLANG_TOOLING_DEPENDENCYSCANNINGTOOL_H diff --git a/clang/lib/CMakeLists.txt b/clang/lib/CMakeLists.txt index e90b009da606a..2fc69e4e4fa6f 100644 --- a/clang/lib/CMakeLists.txt +++ b/clang/lib/CMakeLists.txt @@ -18,6 +18,7 @@ add_subdirectory(Serialization) add_subdirectory(Frontend) add_subdirectory(FrontendTool) add_subdirectory(Tooling) +add_subdirectory(DependencyScanning) add_subdirectory(DirectoryWatcher) add_subdirectory(Index) add_subdirectory(IndexSerialization) diff --git a/clang/lib/Tooling/DependencyScanning/CMakeLists.txt b/clang/lib/DependencyScanning/CMakeLists.txt similarity index 88% rename from clang/lib/Tooling/DependencyScanning/CMakeLists.txt rename to clang/lib/DependencyScanning/CMakeLists.txt index 76bdc50097fff..0b46765649a2a 100644 --- a/clang/lib/Tooling/DependencyScanning/CMakeLists.txt +++ b/clang/lib/DependencyScanning/CMakeLists.txt @@ -9,7 +9,7 @@ add_clang_library(clangDependencyScanning DependencyScanningFilesystem.cpp DependencyScanningService.cpp DependencyScanningWorker.cpp - DependencyScanningTool.cpp + DependencyScanningUtils.cpp DependencyScannerImpl.cpp InProcessModuleCache.cpp ModuleDepCollector.cpp @@ -18,9 +18,7 @@ add_clang_library(clangDependencyScanning ClangDriverOptions LINK_LIBS - clangAST clangBasic - clangDriver clangFrontend clangLex clangSerialization diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp similarity index 83% rename from clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp rename to clang/lib/DependencyScanning/DependencyScannerImpl.cpp index 657547d299abd..11360bf34e9ff 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScannerImpl.cpp +++ b/clang/lib/DependencyScanning/DependencyScannerImpl.cpp @@ -6,17 +6,16 @@ // //===----------------------------------------------------------------------===// -#include "DependencyScannerImpl.h" +#include "clang/DependencyScanning/DependencyScannerImpl.h" #include "clang/Basic/DiagnosticFrontend.h" #include "clang/Basic/DiagnosticSerialization.h" +#include "clang/DependencyScanning/DependencyScanningWorker.h" #include "clang/Driver/Driver.h" #include "clang/Frontend/FrontendActions.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/TargetParser/Host.h" using namespace clang; -using namespace tooling; using namespace dependencies; namespace { @@ -380,42 +379,6 @@ DignosticsEngineWithDiagOpts::DignosticsEngineWithDiagOpts( /*ShouldOwnClient=*/false); } -std::pair, std::unique_ptr> -dependencies::buildCompilation(ArrayRef ArgStrs, - DiagnosticsEngine &Diags, - IntrusiveRefCntPtr FS, - llvm::BumpPtrAllocator &Alloc) { - SmallVector Argv; - Argv.reserve(ArgStrs.size()); - for (const std::string &Arg : ArgStrs) - Argv.push_back(Arg.c_str()); - - std::unique_ptr Driver = std::make_unique( - Argv[0], llvm::sys::getDefaultTargetTriple(), Diags, - "clang LLVM compiler", FS); - Driver->setTitle("clang_based_tool"); - - bool CLMode = driver::IsClangCL( - driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1))); - - if (llvm::Error E = - driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) { - Diags.Report(diag::err_drv_expand_response_file) - << llvm::toString(std::move(E)); - return std::make_pair(nullptr, nullptr); - } - - std::unique_ptr Compilation( - Driver->BuildCompilation(Argv)); - if (!Compilation) - return std::make_pair(nullptr, nullptr); - - if (Compilation->containsError()) - return std::make_pair(nullptr, nullptr); - - return std::make_pair(std::move(Driver), std::move(Compilation)); -} - std::unique_ptr dependencies::createCompilerInvocation(ArrayRef CommandLine, DiagnosticsEngine &Diags) { @@ -431,62 +394,6 @@ dependencies::createCompilerInvocation(ArrayRef CommandLine, return Invocation; } -std::pair, std::vector> -dependencies::initVFSForTUBufferScanning( - IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, StringRef WorkingDirectory, - llvm::MemoryBufferRef TUBuffer) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - IntrusiveRefCntPtr ModifiedFS; - auto OverlayFS = - llvm::makeIntrusiveRefCnt(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - auto InputPath = TUBuffer.getBufferIdentifier(); - InMemoryFS->addFile( - InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); - IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; - - OverlayFS->pushOverlay(InMemoryOverlay); - ModifiedFS = OverlayFS; - std::vector ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(InputPath); - - return std::make_pair(ModifiedFS, ModifiedCommandLine); -} - -std::pair, - std::vector> -dependencies::initVFSForByNameScanning( - IntrusiveRefCntPtr BaseFS, - ArrayRef CommandLine, StringRef WorkingDirectory, - StringRef ModuleName) { - // Reset what might have been modified in the previous worker invocation. - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - - // If we're scanning based on a module name alone, we don't expect the client - // to provide us with an input file. However, the driver really wants to have - // one. Let's just make it up to make the driver happy. - auto OverlayFS = - llvm::makeIntrusiveRefCnt(BaseFS); - auto InMemoryFS = llvm::makeIntrusiveRefCnt(); - InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); - SmallString<128> FakeInputPath; - // TODO: We should retry the creation if the path already exists. - llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath, - /*MakeAbsolute=*/false); - InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); - IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; - OverlayFS->pushOverlay(InMemoryOverlay); - - std::vector ModifiedCommandLine(CommandLine); - ModifiedCommandLine.emplace_back(FakeInputPath); - - return std::make_pair(OverlayFS, ModifiedCommandLine); -} - bool dependencies::initializeScanCompilerInstance( CompilerInstance &ScanInstance, IntrusiveRefCntPtr FS, @@ -707,39 +614,20 @@ bool DependencyScanningAction::runInvocation( return Result; } -bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) { - if (DC) { - DiagConsumer = DC; - } else { - DiagPrinterWithOS = - std::make_unique(CommandLine); - DiagConsumer = &DiagPrinterWithOS->DiagPrinter; - } - - std::tie(OverlayFS, CommandLine) = initVFSForByNameScanning( - Worker.BaseFS, CommandLine, CWD, "ScanningByName"); - - DiagEngineWithCmdAndOpts = std::make_unique( - CommandLine, OverlayFS, *DiagConsumer); +bool CompilerInstanceWithContext::initialize( + std::unique_ptr DiagEngineWithDiagOpts, + IntrusiveRefCntPtr FS) { + assert(DiagEngineWithDiagOpts && "Valid diagnostics engine required!"); + DiagEngineWithCmdAndOpts = std::move(DiagEngineWithDiagOpts); + DiagConsumer = DiagEngineWithCmdAndOpts->DiagEngine->getClient(); - std::tie(Driver, Compilation) = buildCompilation( - CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS, Alloc); - - if (!Compilation) - return false; + // Reset what might have been modified in the previous worker invocation. + Worker.BaseFS->setCurrentWorkingDirectory(CWD); - assert(Compilation->getJobs().size() && - "Must have a job list of non-zero size"); - const driver::Command &Command = *(Compilation->getJobs().begin()); - const auto &CommandArgs = Command.getArguments(); - assert(!CommandArgs.empty() && "Cannot have a command with 0 args"); - assert(StringRef(CommandArgs[0]) == "-cc1" && "Requires a cc1 job."); - OriginalInvocation = std::make_unique(); - - if (!CompilerInvocation::CreateFromArgs(*OriginalInvocation, CommandArgs, - *DiagEngineWithCmdAndOpts->DiagEngine, - Command.getExecutable())) { - DiagEngineWithCmdAndOpts->DiagEngine->Report( + OriginalInvocation = createCompilerInvocation( + CommandLine, *DiagEngineWithCmdAndOpts->DiagEngine); + if (!OriginalInvocation) { + this->DiagEngineWithCmdAndOpts->DiagEngine->Report( diag::err_fe_expected_compiler_job) << llvm::join(CommandLine, " "); return false; @@ -757,7 +645,7 @@ bool CompilerInstanceWithContext::initialize(DiagnosticConsumer *DC) { auto &CI = *CIPtr; if (!initializeScanCompilerInstance( - CI, OverlayFS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(), + CI, FS, DiagEngineWithCmdAndOpts->DiagEngine->getClient(), Worker.Service, Worker.DepFS)) return false; @@ -809,7 +697,7 @@ bool CompilerInstanceWithContext::computeDependencies( // file. In this case, we call BeginSourceFile to initialize. std::unique_ptr Action = std::make_unique(); - auto InputFile = CI.getFrontendOpts().Inputs.begin(); + auto *InputFile = CI.getFrontendOpts().Inputs.begin(); bool ActionBeginSucceeded = Action->BeginSourceFile(CI, *InputFile); assert(ActionBeginSucceeded && "Action BeginSourceFile must succeed"); (void)ActionBeginSucceeded; @@ -870,11 +758,3 @@ bool CompilerInstanceWithContext::finalize() { DiagConsumer->finish(); return true; } - -llvm::Error CompilerInstanceWithContext::handleReturnStatus(bool Success) { - assert(DiagPrinterWithOS && "Must use the default DiagnosticConsumer."); - return Success ? llvm::Error::success() - : llvm::make_error( - DiagPrinterWithOS->DiagnosticsOS.str(), - llvm::inconvertibleErrorCode()); -} diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/DependencyScanning/DependencyScanningFilesystem.cpp similarity index 99% rename from clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp rename to clang/lib/DependencyScanning/DependencyScanningFilesystem.cpp index 266944ee730cb..24a794e4a6a22 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningFilesystem.cpp @@ -1,4 +1,4 @@ -//===- DependencyScanningFilesystem.cpp - clang-scan-deps fs --------------===// +//===- DependencyScanningFilesystem.cpp - Optimized Scanning FS -----------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,13 +6,12 @@ // //===----------------------------------------------------------------------===// -#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/DependencyScanning/DependencyScanningFilesystem.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Threading.h" #include using namespace clang; -using namespace tooling; using namespace dependencies; llvm::ErrorOr diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/DependencyScanning/DependencyScanningService.cpp similarity index 82% rename from clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp rename to clang/lib/DependencyScanning/DependencyScanningService.cpp index 7f40c99f07287..72f359e56d116 100644 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp +++ b/clang/lib/DependencyScanning/DependencyScanningService.cpp @@ -1,4 +1,4 @@ -//===- DependencyScanningService.cpp - clang-scan-deps service ------------===// +//===- DependencyScanningService.cpp - Scanning Service -------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,10 +6,9 @@ // //===----------------------------------------------------------------------===// -#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/DependencyScanning/DependencyScanningService.h" using namespace clang; -using namespace tooling; using namespace dependencies; DependencyScanningService::DependencyScanningService( diff --git a/clang/lib/DependencyScanning/DependencyScanningUtils.cpp b/clang/lib/DependencyScanning/DependencyScanningUtils.cpp new file mode 100644 index 0000000000000..e27c597a14fcc --- /dev/null +++ b/clang/lib/DependencyScanning/DependencyScanningUtils.cpp @@ -0,0 +1,38 @@ +//===- DependencyScanningUtils.cpp - Common Scanning Utilities ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DependencyScanning/DependencyScanningUtils.h" + +using namespace clang; +using namespace dependencies; + +TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { + TranslationUnitDeps TU; + + TU.ID.ContextHash = std::move(ContextHash); + TU.ID.ModuleName = std::move(ModuleName); + TU.NamedModuleDeps = std::move(NamedModuleDeps); + TU.FileDeps = std::move(Dependencies); + TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps); + TU.VisibleModules = std::move(VisibleModules); + TU.Commands = std::move(Commands); + + for (auto &&M : ClangModuleDeps) { + auto &MD = M.second; + // TODO: Avoid handleModuleDependency even being called for modules + // we've already seen. + if (AlreadySeen.count(M.first)) + continue; + TU.ModuleGraph.push_back(std::move(MD)); + } + TU.ClangModuleDeps = std::move(DirectModuleDeps); + + return TU; +} + +CallbackActionController::~CallbackActionController() {} diff --git a/clang/lib/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp new file mode 100644 index 0000000000000..34182f3f5087f --- /dev/null +++ b/clang/lib/DependencyScanning/DependencyScanningWorker.cpp @@ -0,0 +1,147 @@ +//===- DependencyScanningWorker.cpp - Thread-Safe Scanning Worker ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticFrontend.h" +#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Tool.h" +#include "clang/Serialization/ObjectFilePCHContainerReader.h" + +using namespace clang; +using namespace dependencies; + +DependencyScanningWorker::DependencyScanningWorker( + DependencyScanningService &Service, + llvm::IntrusiveRefCntPtr FS) + : Service(Service) { + PCHContainerOps = std::make_shared(); + // We need to read object files from PCH built outside the scanner. + PCHContainerOps->registerReader( + std::make_unique()); + // The scanner itself writes only raw ast files. + PCHContainerOps->registerWriter(std::make_unique()); + + if (Service.shouldTraceVFS()) + FS = llvm::makeIntrusiveRefCnt(std::move(FS)); + + switch (Service.getMode()) { + case ScanningMode::DependencyDirectivesScan: + DepFS = llvm::makeIntrusiveRefCnt( + Service.getSharedCache(), FS); + BaseFS = DepFS; + break; + case ScanningMode::CanonicalPreprocessing: + DepFS = nullptr; + BaseFS = FS; + break; + } +} + +DependencyScanningWorker::~DependencyScanningWorker() = default; +DependencyActionController::~DependencyActionController() = default; + +static bool createAndRunToolInvocation( + const std::vector &CommandLine, + DependencyScanningAction &Action, + IntrusiveRefCntPtr FS, + std::shared_ptr &PCHContainerOps, + DiagnosticsEngine &Diags, DependencyConsumer &Consumer) { + auto Invocation = createCompilerInvocation(CommandLine, Diags); + if (!Invocation) + return false; + + if (!Action.runInvocation(std::move(Invocation), std::move(FS), + PCHContainerOps, Diags.getClient())) + return false; + + std::vector Args = Action.takeLastCC1Arguments(); + Consumer.handleBuildCommand({CommandLine[0], std::move(Args)}); + return true; +} + +bool DependencyScanningWorker::scanDependencies( + StringRef WorkingDirectory, ArrayRef> CommandLines, + DependencyConsumer &Consumer, DependencyActionController &Controller, + DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr FS) { + DependencyScanningAction Action(Service, WorkingDirectory, Consumer, + Controller, DepFS); + + const bool Success = llvm::all_of(CommandLines, [&](const auto &Cmd) { + if (StringRef(Cmd[1]) != "-cc1") { + // Non-clang command. Just pass through to the dependency consumer. + Consumer.handleBuildCommand({Cmd.front(), {Cmd.begin() + 1, Cmd.end()}}); + return true; + } + // Create an invocation that uses the underlying file + // system to ensure that any file system requests that + // are made by the driver do not go through the + // dependency scanning filesystem. + return createAndRunToolInvocation(Cmd, Action, FS, PCHContainerOps, Diags, + Consumer); + }); + + // Ensure finish() is called even if we never reached ExecuteAction(). + if (!Action.hasDiagConsumerFinished()) + Diags.getClient()->finish(); + + return Success && Action.hasScanned(); +} + +bool DependencyScanningWorker::computeDependencies( + StringRef WorkingDirectory, const std::vector &CommandLine, + DependencyConsumer &DepConsumer, DependencyActionController &Controller, + DiagnosticConsumer &DiagConsumer, + llvm::IntrusiveRefCntPtr ScanFS) { + auto FinalFS = ScanFS == nullptr ? BaseFS : ScanFS; + DignosticsEngineWithDiagOpts DiagEngineWithDiagOpts(CommandLine, FinalFS, + DiagConsumer); + return computeDependencies( + WorkingDirectory, ArrayRef>{CommandLine}, + DepConsumer, Controller, *DiagEngineWithDiagOpts.DiagEngine, ScanFS); +} + +bool DependencyScanningWorker::computeDependencies( + StringRef WorkingDirectory, ArrayRef> CommandLines, + DependencyConsumer &DepConsumer, DependencyActionController &Controller, + DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr ScanFS) { + auto FinalFS = ScanFS == nullptr ? BaseFS : ScanFS; + return scanDependencies(WorkingDirectory, CommandLines, DepConsumer, + Controller, Diags, FinalFS); +} + +bool DependencyScanningWorker::initializeCompilerInstanceWithContext( + StringRef CWD, ArrayRef CommandLine, DiagnosticConsumer &DC) { + auto DiagEngineWithCmdAndOpts = + std::make_unique(CommandLine, BaseFS, DC); + return initializeCompilerInstanceWithContext( + CWD, CommandLine, std::move(DiagEngineWithCmdAndOpts), BaseFS); +} + +bool DependencyScanningWorker::initializeCompilerInstanceWithContext( + StringRef CWD, ArrayRef CommandLine, + std::unique_ptr DiagEngineWithDiagOpts, + IntrusiveRefCntPtr FS) { + CIWithContext = + std::make_unique(*this, CWD, CommandLine); + return CIWithContext->initialize(std::move(DiagEngineWithDiagOpts), FS); +} + +bool DependencyScanningWorker::computeDependenciesByNameWithContext( + StringRef ModuleName, DependencyConsumer &Consumer, + DependencyActionController &Controller) { + assert(CIWithContext && "CompilerInstance with context required!"); + return CIWithContext->computeDependencies(ModuleName, Consumer, Controller); +} + +bool DependencyScanningWorker::finalizeCompilerInstanceWithContext() { + return CIWithContext->finalize(); +} diff --git a/clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp b/clang/lib/DependencyScanning/InProcessModuleCache.cpp similarity index 95% rename from clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp rename to clang/lib/DependencyScanning/InProcessModuleCache.cpp index d1e543b438225..1dd2d34032a96 100644 --- a/clang/lib/Tooling/DependencyScanning/InProcessModuleCache.cpp +++ b/clang/lib/DependencyScanning/InProcessModuleCache.cpp @@ -1,4 +1,4 @@ -//===----------------------------------------------------------------------===// +//===- InProcessModuleCache.cpp - Implicit Module Cache ---------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// -#include "clang/Tooling/DependencyScanning/InProcessModuleCache.h" +#include "clang/DependencyScanning/InProcessModuleCache.h" #include "clang/Serialization/InMemoryModuleCache.h" #include "llvm/Support/AdvisoryLock.h" @@ -15,7 +15,6 @@ #include using namespace clang; -using namespace tooling; using namespace dependencies; namespace { diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/DependencyScanning/ModuleDepCollector.cpp similarity index 99% rename from clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp rename to clang/lib/DependencyScanning/ModuleDepCollector.cpp index 3a99f8c882b8f..39bd2e2ab0032 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/DependencyScanning/ModuleDepCollector.cpp @@ -6,18 +6,17 @@ // //===----------------------------------------------------------------------===// -#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" +#include "clang/DependencyScanning/ModuleDepCollector.h" #include "clang/Basic/MakeSupport.h" +#include "clang/DependencyScanning/DependencyScanningWorker.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Lex/Preprocessor.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/BLAKE3.h" #include using namespace clang; -using namespace tooling; using namespace dependencies; void ModuleDeps::forEachFileDep(llvm::function_ref Cb) const { diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt index faaa53276d0e6..e39124e105cf3 100644 --- a/clang/lib/Tooling/CMakeLists.txt +++ b/clang/lib/Tooling/CMakeLists.txt @@ -10,7 +10,6 @@ add_subdirectory(Inclusions) add_subdirectory(Refactoring) add_subdirectory(ASTDiff) add_subdirectory(Syntax) -add_subdirectory(DependencyScanning) add_subdirectory(Transformer) add_clang_library(clangTooling @@ -18,6 +17,7 @@ add_clang_library(clangTooling ArgumentsAdjusters.cpp CommonOptionsParser.cpp CompilationDatabase.cpp + DependencyScanningTool.cpp Execution.cpp ExpandResponseFilesCompilationDatabase.cpp FileMatchTrie.cpp @@ -39,7 +39,9 @@ add_clang_library(clangTooling clangAST clangASTMatchers clangBasic + clangDependencyScanning clangDriver + clangDependencyScanning clangOptions clangFormat clangFrontend diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp deleted file mode 100644 index a1f2db7a471be..0000000000000 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp +++ /dev/null @@ -1,228 +0,0 @@ -//===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" -#include "clang/Frontend/Utils.h" -#include - -using namespace clang; -using namespace tooling; -using namespace dependencies; - -DependencyScanningTool::DependencyScanningTool( - DependencyScanningService &Service, - llvm::IntrusiveRefCntPtr FS) - : Worker(Service, std::move(FS)) {} - -namespace { -/// Prints out all of the gathered dependencies into a string. -class MakeDependencyPrinterConsumer : public DependencyConsumer { -public: - void handleBuildCommand(Command) override {} - - void - handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override { - this->Opts = std::make_unique(Opts); - } - - void handleFileDependency(StringRef File) override { - Dependencies.push_back(std::string(File)); - } - - // These are ignored for the make format as it can't support the full - // set of deps, and handleFileDependency handles enough for implicitly - // built modules to work. - void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {} - void handleModuleDependency(ModuleDeps MD) override {} - void handleDirectModuleDependency(ModuleID ID) override {} - void handleVisibleModule(std::string ModuleName) override {} - void handleContextHash(std::string Hash) override {} - - void printDependencies(std::string &S) { - assert(Opts && "Handled dependency output options."); - - class DependencyPrinter : public DependencyFileGenerator { - public: - DependencyPrinter(DependencyOutputOptions &Opts, - ArrayRef Dependencies) - : DependencyFileGenerator(Opts) { - for (const auto &Dep : Dependencies) - addDependency(Dep); - } - - void printDependencies(std::string &S) { - llvm::raw_string_ostream OS(S); - outputDependencyFile(OS); - } - }; - - DependencyPrinter Generator(*Opts, Dependencies); - Generator.printDependencies(S); - } - -protected: - std::unique_ptr Opts; - std::vector Dependencies; -}; -} // anonymous namespace - -llvm::Expected DependencyScanningTool::getDependencyFile( - const std::vector &CommandLine, StringRef CWD) { - MakeDependencyPrinterConsumer Consumer; - CallbackActionController Controller(nullptr); - auto Result = - Worker.computeDependencies(CWD, CommandLine, Consumer, Controller); - if (Result) - return std::move(Result); - std::string Output; - Consumer.printDependencies(Output); - return Output; -} - -llvm::Expected DependencyScanningTool::getP1689ModuleDependencyFile( - const CompileCommand &Command, StringRef CWD, std::string &MakeformatOutput, - std::string &MakeformatOutputPath) { - class P1689ModuleDependencyPrinterConsumer - : public MakeDependencyPrinterConsumer { - public: - P1689ModuleDependencyPrinterConsumer(P1689Rule &Rule, - const CompileCommand &Command) - : Filename(Command.Filename), Rule(Rule) { - Rule.PrimaryOutput = Command.Output; - } - - void handleProvidedAndRequiredStdCXXModules( - std::optional Provided, - std::vector Requires) override { - Rule.Provides = Provided; - if (Rule.Provides) - Rule.Provides->SourcePath = Filename.str(); - Rule.Requires = Requires; - } - - StringRef getMakeFormatDependencyOutputPath() { - if (Opts->OutputFormat != DependencyOutputFormat::Make) - return {}; - return Opts->OutputFile; - } - - private: - StringRef Filename; - P1689Rule &Rule; - }; - - class P1689ActionController : public DependencyActionController { - public: - // The lookupModuleOutput is for clang modules. P1689 format don't need it. - std::string lookupModuleOutput(const ModuleDeps &, - ModuleOutputKind Kind) override { - return ""; - } - }; - - P1689Rule Rule; - P1689ModuleDependencyPrinterConsumer Consumer(Rule, Command); - P1689ActionController Controller; - auto Result = Worker.computeDependencies(CWD, Command.CommandLine, Consumer, - Controller); - if (Result) - return std::move(Result); - - MakeformatOutputPath = Consumer.getMakeFormatDependencyOutputPath(); - if (!MakeformatOutputPath.empty()) - Consumer.printDependencies(MakeformatOutput); - return Rule; -} - -llvm::Expected -DependencyScanningTool::getTranslationUnitDependencies( - const std::vector &CommandLine, StringRef CWD, - const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput, - std::optional TUBuffer) { - FullDependencyConsumer Consumer(AlreadySeen); - CallbackActionController Controller(LookupModuleOutput); - llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer, - Controller, TUBuffer); - - if (Result) - return std::move(Result); - return Consumer.takeTranslationUnitDeps(); -} - -llvm::Expected -DependencyScanningTool::getModuleDependencies( - StringRef ModuleName, const std::vector &CommandLine, - StringRef CWD, const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput) { - FullDependencyConsumer Consumer(AlreadySeen); - CallbackActionController Controller(LookupModuleOutput); - if (auto Error = - Worker.initializeCompilerInstanceWithContextOrError(CWD, CommandLine)) - return std::move(Error); - - auto Result = Worker.computeDependenciesByNameWithContextOrError( - ModuleName, Consumer, Controller); - - if (auto Error = Worker.finalizeCompilerInstanceWithContextOrError()) - return std::move(Error); - - if (Result) - return std::move(Result); - - return Consumer.takeTranslationUnitDeps(); -} - -llvm::Error DependencyScanningTool::initializeCompilerInstanceWithContext( - StringRef CWD, const std::vector &CommandLine) { - return Worker.initializeCompilerInstanceWithContextOrError(CWD, CommandLine); -} - -llvm::Expected -DependencyScanningTool::computeDependenciesByNameWithContext( - StringRef ModuleName, const llvm::DenseSet &AlreadySeen, - LookupModuleOutputCallback LookupModuleOutput) { - FullDependencyConsumer Consumer(AlreadySeen); - CallbackActionController Controller(LookupModuleOutput); - llvm::Error Result = Worker.computeDependenciesByNameWithContextOrError( - ModuleName, Consumer, Controller); - if (Result) - return std::move(Result); - - return Consumer.takeTranslationUnitDeps(); -} - -llvm::Error DependencyScanningTool::finalizeCompilerInstanceWithContext() { - return Worker.finalizeCompilerInstanceWithContextOrError(); -} - -TranslationUnitDeps FullDependencyConsumer::takeTranslationUnitDeps() { - TranslationUnitDeps TU; - - TU.ID.ContextHash = std::move(ContextHash); - TU.ID.ModuleName = std::move(ModuleName); - TU.NamedModuleDeps = std::move(NamedModuleDeps); - TU.FileDeps = std::move(Dependencies); - TU.PrebuiltModuleDeps = std::move(PrebuiltModuleDeps); - TU.VisibleModules = std::move(VisibleModules); - TU.Commands = std::move(Commands); - - for (auto &&M : ClangModuleDeps) { - auto &MD = M.second; - // TODO: Avoid handleModuleDependency even being called for modules - // we've already seen. - if (AlreadySeen.count(M.first)) - continue; - TU.ModuleGraph.push_back(std::move(MD)); - } - TU.ClangModuleDeps = std::move(DirectModuleDeps); - - return TU; -} - -CallbackActionController::~CallbackActionController() {} diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp deleted file mode 100644 index 0bc17f9c80605..0000000000000 --- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp +++ /dev/null @@ -1,210 +0,0 @@ -//===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" -#include "DependencyScannerImpl.h" -#include "clang/Basic/DiagnosticFrontend.h" -#include "clang/Driver/Driver.h" -#include "clang/Driver/Tool.h" - -using namespace clang; -using namespace tooling; -using namespace dependencies; - -DependencyScanningWorker::DependencyScanningWorker( - DependencyScanningService &Service, - llvm::IntrusiveRefCntPtr FS) - : Service(Service) { - PCHContainerOps = std::make_shared(); - // We need to read object files from PCH built outside the scanner. - PCHContainerOps->registerReader( - std::make_unique()); - // The scanner itself writes only raw ast files. - PCHContainerOps->registerWriter(std::make_unique()); - - if (Service.shouldTraceVFS()) - FS = llvm::makeIntrusiveRefCnt(std::move(FS)); - - switch (Service.getMode()) { - case ScanningMode::DependencyDirectivesScan: - DepFS = llvm::makeIntrusiveRefCnt( - Service.getSharedCache(), FS); - BaseFS = DepFS; - break; - case ScanningMode::CanonicalPreprocessing: - DepFS = nullptr; - BaseFS = FS; - break; - } -} - -DependencyScanningWorker::~DependencyScanningWorker() = default; -DependencyActionController::~DependencyActionController() = default; - -llvm::Error DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - std::optional TUBuffer) { - // Capture the emitted diagnostics and report them to the client - // in the case of a failure. - TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine); - - if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DiagPrinterWithOS.DiagPrinter, TUBuffer)) - return llvm::Error::success(); - return llvm::make_error( - DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); -} - -static bool forEachDriverJob( - ArrayRef ArgStrs, DiagnosticsEngine &Diags, - IntrusiveRefCntPtr FS, - llvm::function_ref Callback) { - // Compilation holds a non-owning a reference to the Driver, hence we need to - // keep the Driver alive when we use Compilation. Arguments to commands may be - // owned by Alloc when expanded from response files. - llvm::BumpPtrAllocator Alloc; - auto [Driver, Compilation] = buildCompilation(ArgStrs, Diags, FS, Alloc); - if (!Compilation) - return false; - for (const driver::Command &Job : Compilation->getJobs()) { - if (!Callback(Job)) - return false; - } - return true; -} - -static bool createAndRunToolInvocation( - const std::vector &CommandLine, - DependencyScanningAction &Action, - IntrusiveRefCntPtr FS, - std::shared_ptr &PCHContainerOps, - DiagnosticsEngine &Diags, DependencyConsumer &Consumer) { - auto Invocation = createCompilerInvocation(CommandLine, Diags); - if (!Invocation) - return false; - - if (!Action.runInvocation(std::move(Invocation), std::move(FS), - PCHContainerOps, Diags.getClient())) - return false; - - std::vector Args = Action.takeLastCC1Arguments(); - Consumer.handleBuildCommand({CommandLine[0], std::move(Args)}); - return true; -} - -bool DependencyScanningWorker::scanDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, - llvm::IntrusiveRefCntPtr FS) { - DignosticsEngineWithDiagOpts DiagEngineWithCmdAndOpts(CommandLine, FS, DC); - DependencyScanningAction Action(Service, WorkingDirectory, Consumer, - Controller, DepFS); - - bool Success = false; - if (CommandLine[1] == "-cc1") { - Success = createAndRunToolInvocation( - CommandLine, Action, FS, PCHContainerOps, - *DiagEngineWithCmdAndOpts.DiagEngine, Consumer); - } else { - Success = forEachDriverJob( - CommandLine, *DiagEngineWithCmdAndOpts.DiagEngine, FS, - [&](const driver::Command &Cmd) { - if (StringRef(Cmd.getCreator().getName()) != "clang") { - // Non-clang command. Just pass through to the dependency - // consumer. - Consumer.handleBuildCommand( - {Cmd.getExecutable(), - {Cmd.getArguments().begin(), Cmd.getArguments().end()}}); - return true; - } - - // Insert -cc1 comand line options into Argv - std::vector Argv; - Argv.push_back(Cmd.getExecutable()); - llvm::append_range(Argv, Cmd.getArguments()); - - // Create an invocation that uses the underlying file - // system to ensure that any file system requests that - // are made by the driver do not go through the - // dependency scanning filesystem. - return createAndRunToolInvocation( - std::move(Argv), Action, FS, PCHContainerOps, - *DiagEngineWithCmdAndOpts.DiagEngine, Consumer); - }); - } - - if (Success && !Action.hasScanned()) - DiagEngineWithCmdAndOpts.DiagEngine->Report( - diag::err_fe_expected_compiler_job) - << llvm::join(CommandLine, " "); - - // Ensure finish() is called even if we never reached ExecuteAction(). - if (!Action.hasDiagConsumerFinished()) - DC.finish(); - - return Success && Action.hasScanned(); -} - -bool DependencyScanningWorker::computeDependencies( - StringRef WorkingDirectory, const std::vector &CommandLine, - DependencyConsumer &Consumer, DependencyActionController &Controller, - DiagnosticConsumer &DC, std::optional TUBuffer) { - if (TUBuffer) { - auto [FinalFS, FinalCommandLine] = initVFSForTUBufferScanning( - BaseFS, CommandLine, WorkingDirectory, *TUBuffer); - return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer, - Controller, DC, FinalFS); - } else { - BaseFS->setCurrentWorkingDirectory(WorkingDirectory); - return scanDependencies(WorkingDirectory, CommandLine, Consumer, Controller, - DC, BaseFS); - } -} - -llvm::Error -DependencyScanningWorker::initializeCompilerInstanceWithContextOrError( - StringRef CWD, const std::vector &CommandLine) { - bool Success = initializeCompilerInstanceWithContext(CWD, CommandLine); - return CIWithContext->handleReturnStatus(Success); -} - -llvm::Error -DependencyScanningWorker::computeDependenciesByNameWithContextOrError( - StringRef ModuleName, DependencyConsumer &Consumer, - DependencyActionController &Controller) { - bool Success = - computeDependenciesByNameWithContext(ModuleName, Consumer, Controller); - return CIWithContext->handleReturnStatus(Success); -} - -llvm::Error -DependencyScanningWorker::finalizeCompilerInstanceWithContextOrError() { - bool Success = finalizeCompilerInstance(); - return CIWithContext->handleReturnStatus(Success); -} - -bool DependencyScanningWorker::initializeCompilerInstanceWithContext( - StringRef CWD, const std::vector &CommandLine, - DiagnosticConsumer *DC) { - CIWithContext = - std::make_unique(*this, CWD, CommandLine); - return CIWithContext->initialize(DC); -} - -bool DependencyScanningWorker::computeDependenciesByNameWithContext( - StringRef ModuleName, DependencyConsumer &Consumer, - DependencyActionController &Controller) { - assert(CIWithContext && "CompilerInstance with context required!"); - return CIWithContext->computeDependencies(ModuleName, Consumer, Controller); -} - -bool DependencyScanningWorker::finalizeCompilerInstance() { - return CIWithContext->finalize(); -} diff --git a/clang/lib/Tooling/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanningTool.cpp new file mode 100644 index 0000000000000..6cfa15d0cbe77 --- /dev/null +++ b/clang/lib/Tooling/DependencyScanningTool.cpp @@ -0,0 +1,425 @@ +//===- DependencyScanningTool.cpp - clang-scan-deps service ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/DependencyScanningTool.h" +#include "clang/Basic/DiagnosticFrontend.h" +#include "clang/DependencyScanning/DependencyScannerImpl.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Tool.h" +#include "clang/Frontend/Utils.h" +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include "llvm/TargetParser/Host.h" +#include + +using namespace clang; +using namespace tooling; +using namespace clang::dependencies; +using namespace clang::tooling::dependencies; + +DependencyScanningTool::DependencyScanningTool( + DependencyScanningService &Service, + llvm::IntrusiveRefCntPtr FS) + : Worker(Service, std::move(FS)) {} + +namespace { +/// Prints out all of the gathered dependencies into a string. +class MakeDependencyPrinterConsumer : public DependencyConsumer { +public: + void handleBuildCommand(Command) override {} + + void + handleDependencyOutputOpts(const DependencyOutputOptions &Opts) override { + this->Opts = std::make_unique(Opts); + } + + void handleFileDependency(StringRef File) override { + Dependencies.push_back(std::string(File)); + } + + // These are ignored for the make format as it can't support the full + // set of deps, and handleFileDependency handles enough for implicitly + // built modules to work. + void handlePrebuiltModuleDependency(PrebuiltModuleDep PMD) override {} + void handleModuleDependency(ModuleDeps MD) override {} + void handleDirectModuleDependency(ModuleID ID) override {} + void handleVisibleModule(std::string ModuleName) override {} + void handleContextHash(std::string Hash) override {} + + void printDependencies(std::string &S) { + assert(Opts && "Handled dependency output options."); + + class DependencyPrinter : public DependencyFileGenerator { + public: + DependencyPrinter(DependencyOutputOptions &Opts, + ArrayRef Dependencies) + : DependencyFileGenerator(Opts) { + for (const auto &Dep : Dependencies) + addDependency(Dep); + } + + void printDependencies(std::string &S) { + llvm::raw_string_ostream OS(S); + outputDependencyFile(OS); + } + }; + + DependencyPrinter Generator(*Opts, Dependencies); + Generator.printDependencies(S); + } + +protected: + std::unique_ptr Opts; + std::vector Dependencies; +}; +} // anonymous namespace + +static std::pair, + std::unique_ptr> +buildCompilation(ArrayRef CommandLine, DiagnosticsEngine &Diags, + IntrusiveRefCntPtr FS, + llvm::BumpPtrAllocator &Alloc) { + SmallVector Argv; + Argv.reserve(CommandLine.size()); + for (const std::string &Arg : CommandLine) + Argv.push_back(Arg.c_str()); + + std::unique_ptr Driver = std::make_unique( + Argv[0], llvm::sys::getDefaultTargetTriple(), Diags, + "clang LLVM compiler", FS); + Driver->setTitle("clang_based_tool"); + + bool CLMode = driver::IsClangCL( + driver::getDriverMode(Argv[0], ArrayRef(Argv).slice(1))); + + if (llvm::Error E = + driver::expandResponseFiles(Argv, CLMode, Alloc, FS.get())) { + Diags.Report(diag::err_drv_expand_response_file) + << llvm::toString(std::move(E)); + return std::make_pair(nullptr, nullptr); + } + + std::unique_ptr Compilation( + Driver->BuildCompilation(Argv)); + if (!Compilation) + return std::make_pair(nullptr, nullptr); + + if (Compilation->containsError()) + return std::make_pair(nullptr, nullptr); + + if (Compilation->getJobs().empty()) { + Diags.Report(diag::err_fe_expected_compiler_job) + << llvm::join(CommandLine, " "); + return std::make_pair(nullptr, nullptr); + } + + return std::make_pair(std::move(Driver), std::move(Compilation)); +} + +/// Constructs the full -cc1 command line, including executable, for the given +/// driver \c Cmd. +static std::vector +buildCC1CommandLine(const driver::Command &Cmd) { + const auto &Args = Cmd.getArguments(); + std::vector Out; + Out.reserve(Args.size() + 1); + Out.emplace_back(Cmd.getExecutable()); + llvm::append_range(Out, Args); + return Out; +} + +static std::pair, + std::vector> +initVFSForTUBufferScanning(IntrusiveRefCntPtr BaseFS, + ArrayRef CommandLine, + StringRef WorkingDirectory, + llvm::MemoryBufferRef TUBuffer) { + // Reset what might have been modified in the previous worker invocation. + BaseFS->setCurrentWorkingDirectory(WorkingDirectory); + + IntrusiveRefCntPtr ModifiedFS; + auto OverlayFS = + llvm::makeIntrusiveRefCnt(BaseFS); + auto InMemoryFS = llvm::makeIntrusiveRefCnt(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + auto InputPath = TUBuffer.getBufferIdentifier(); + InMemoryFS->addFile( + InputPath, 0, llvm::MemoryBuffer::getMemBufferCopy(TUBuffer.getBuffer())); + IntrusiveRefCntPtr InMemoryOverlay = InMemoryFS; + + OverlayFS->pushOverlay(InMemoryOverlay); + std::vector ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(InputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} + +static std::pair, + std::vector> +initVFSForByNameScanning(IntrusiveRefCntPtr BaseFS, + ArrayRef CommandLine, + StringRef WorkingDirectory, StringRef ModuleName) { + // If we're scanning based on a module name alone, we don't expect the client + // to provide us with an input file. However, the driver really wants to have + // one. Let's just make it up to make the driver happy. + auto OverlayFS = + llvm::makeIntrusiveRefCnt(BaseFS); + // Reset what might have been modified in the previous worker invocation. + OverlayFS->setCurrentWorkingDirectory(WorkingDirectory); + auto InMemoryFS = llvm::makeIntrusiveRefCnt(); + InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory); + SmallString<128> FakeInputPath; + // TODO: We should retry the creation if the path already exists. + llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath, + /*MakeAbsolute=*/false); + InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + OverlayFS->pushOverlay(InMemoryFS); + + std::vector ModifiedCommandLine(CommandLine); + ModifiedCommandLine.emplace_back(FakeInputPath); + + return std::make_pair(OverlayFS, ModifiedCommandLine); +} + +static llvm::Error makeErrorFromDiagnosticsOS( + TextDiagnosticsPrinterWithOutput &DiagPrinterWithOS) { + return llvm::make_error( + DiagPrinterWithOS.DiagnosticsOS.str(), llvm::inconvertibleErrorCode()); +} + +static bool computeDependenciesForDriverCommandLine( + DependencyScanningWorker &Worker, StringRef WorkingDirectory, + ArrayRef CommandLine, DependencyConsumer &Consumer, + DependencyActionController &Controller, DiagnosticConsumer &DiagConsumer, + IntrusiveRefCntPtr ScanFS) { + Worker.getVFS().setCurrentWorkingDirectory(WorkingDirectory); + + DignosticsEngineWithDiagOpts DiagEngineWithDiagOpts( + CommandLine, &Worker.getVFS(), DiagConsumer); + auto &Diags = *DiagEngineWithDiagOpts.DiagEngine; + + // Compilation holds a non-owning a reference to the Driver, hence we need to + // keep the Driver alive when we use Compilation. Arguments to commands may be + // owned by Alloc when expanded from response files. + llvm::BumpPtrAllocator Alloc; + const auto [Driver, Compilation] = buildCompilation( + CommandLine, *DiagEngineWithDiagOpts.DiagEngine, &Worker.getVFS(), Alloc); + if (!Compilation) + return false; + + const auto CC1Commands = llvm::to_vector( + llvm::map_range(Compilation->getJobs(), buildCC1CommandLine)); + + return Worker.computeDependencies(WorkingDirectory, CC1Commands, Consumer, + Controller, Diags, ScanFS); +} + +static llvm::Error computeDependenciesOrError( + DependencyScanningWorker &Worker, StringRef WorkingDirectory, + ArrayRef CommandLine, DependencyConsumer &Consumer, + DependencyActionController &Controller, + std::optional TUBuffer = std::nullopt) { + auto [OverlayFS, FinalCommandLine] = [&]() { + if (TUBuffer) + return initVFSForTUBufferScanning(&Worker.getVFS(), CommandLine, + WorkingDirectory, *TUBuffer); + return std::make_pair( + IntrusiveRefCntPtr(&Worker.getVFS()), + std::vector(CommandLine.begin(), CommandLine.end())); + }(); + + TextDiagnosticsPrinterWithOutput DiagPrinterWithOS(CommandLine); + + const auto IsCC1Input = (FinalCommandLine[1] == "-cc1"); + const auto Success = + IsCC1Input + ? Worker.computeDependencies(WorkingDirectory, FinalCommandLine, + Consumer, Controller, + DiagPrinterWithOS.DiagPrinter, OverlayFS) + : computeDependenciesForDriverCommandLine( + Worker, WorkingDirectory, FinalCommandLine, Consumer, + Controller, DiagPrinterWithOS.DiagPrinter, OverlayFS); + + if (!Success) + return makeErrorFromDiagnosticsOS(DiagPrinterWithOS); + return llvm::Error::success(); +} + +llvm::Expected DependencyScanningTool::getDependencyFile( + const std::vector &CommandLine, StringRef CWD) { + MakeDependencyPrinterConsumer Consumer; + CallbackActionController Controller(nullptr); + if (auto Result = computeDependenciesOrError(Worker, CWD, CommandLine, + Consumer, Controller)) + return std::move(Result); + std::string Output; + Consumer.printDependencies(Output); + return Output; +} + +llvm::Expected DependencyScanningTool::getP1689ModuleDependencyFile( + const CompileCommand &Command, StringRef CWD, std::string &MakeformatOutput, + std::string &MakeformatOutputPath) { + class P1689ModuleDependencyPrinterConsumer + : public MakeDependencyPrinterConsumer { + public: + P1689ModuleDependencyPrinterConsumer(P1689Rule &Rule, + const CompileCommand &Command) + : Filename(Command.Filename), Rule(Rule) { + Rule.PrimaryOutput = Command.Output; + } + + void handleProvidedAndRequiredStdCXXModules( + std::optional Provided, + std::vector Requires) override { + Rule.Provides = Provided; + if (Rule.Provides) + Rule.Provides->SourcePath = Filename.str(); + Rule.Requires = Requires; + } + + StringRef getMakeFormatDependencyOutputPath() { + if (Opts->OutputFormat != DependencyOutputFormat::Make) + return {}; + return Opts->OutputFile; + } + + private: + StringRef Filename; + P1689Rule &Rule; + }; + + class P1689ActionController : public DependencyActionController { + public: + // The lookupModuleOutput is for clang modules. P1689 format don't need it. + std::string lookupModuleOutput(const ModuleDeps &, + ModuleOutputKind Kind) override { + return ""; + } + }; + + P1689Rule Rule; + P1689ModuleDependencyPrinterConsumer Consumer(Rule, Command); + P1689ActionController Controller; + if (auto Result = computeDependenciesOrError(Worker, CWD, Command.CommandLine, + Consumer, Controller)) + return std::move(Result); + + MakeformatOutputPath = Consumer.getMakeFormatDependencyOutputPath(); + if (!MakeformatOutputPath.empty()) + Consumer.printDependencies(MakeformatOutput); + return Rule; +} + +llvm::Expected +DependencyScanningTool::getTranslationUnitDependencies( + const std::vector &CommandLine, StringRef CWD, + const llvm::DenseSet &AlreadySeen, + LookupModuleOutputCallback LookupModuleOutput, + std::optional TUBuffer) { + FullDependencyConsumer Consumer(AlreadySeen); + CallbackActionController Controller(LookupModuleOutput); + if (auto Result = computeDependenciesOrError(Worker, CWD, CommandLine, + Consumer, Controller, TUBuffer)) + return std::move(Result); + return Consumer.takeTranslationUnitDeps(); +} + +llvm::Expected +DependencyScanningTool::getModuleDependencies( + StringRef ModuleName, const std::vector &CommandLine, + StringRef CWD, const llvm::DenseSet &AlreadySeen, + LookupModuleOutputCallback LookupModuleOutput) { + if (auto Error = + initializeCompilerInstanceWithContextOrError(CWD, CommandLine)) + return Error; + + auto Result = computeDependenciesByNameWithContextOrError( + ModuleName, AlreadySeen, LookupModuleOutput); + + if (auto Error = finalizeCompilerInstanceWithContextOrError()) + return Error; + + return Result; +} + +static std::optional> getFirstCC1CommandLine( + ArrayRef CommandLine, DiagnosticsEngine &Diags, + llvm::IntrusiveRefCntPtr ScanFS) { + // Compilation holds a non-owning a reference to the Driver, hence we need to + // keep the Driver alive when we use Compilation. Arguments to commands may be + // owned by Alloc when expanded from response files. + llvm::BumpPtrAllocator Alloc; + const auto [Driver, Compilation] = + buildCompilation(CommandLine, Diags, ScanFS, Alloc); + if (!Compilation) + return std::nullopt; + + const auto IsClangCmd = [](const driver::Command &Cmd) { + return StringRef(Cmd.getCreator().getName()) == "clang"; + }; + const auto CC1CommandLineRange = llvm::map_range( + llvm::make_filter_range(Compilation->getJobs(), IsClangCmd), + buildCC1CommandLine); + + if (CC1CommandLineRange.empty()) + return std::nullopt; + return *CC1CommandLineRange.begin(); +} + +llvm::Error +DependencyScanningTool::initializeCompilerInstanceWithContextOrError( + StringRef CWD, ArrayRef CommandLine) { + // For by name scanning, we allow command lines without an actual input file + // by adding an in-memory placeholder input. + auto OverlayFSAndArgs = initVFSForByNameScanning( + &Worker.getVFS(), CommandLine, CWD, "ScanningByName"); + auto &OverlayFS = OverlayFSAndArgs.first; + const auto &ModifiedCommandLine = OverlayFSAndArgs.second; + + DiagPrinterWithOS = + std::make_unique(CommandLine); + auto DiagEngineWithCmdAndOpts = + std::make_unique( + CommandLine, OverlayFS, DiagPrinterWithOS->DiagPrinter); + + const auto InitWithCommandLine = + [&](ArrayRef CommandLine) -> llvm::Error { + if (Worker.initializeCompilerInstanceWithContext( + CWD, CommandLine, std::move(DiagEngineWithCmdAndOpts), OverlayFS)) + return llvm::Error::success(); + return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS); + }; + + if (CommandLine.size() >= 2 && CommandLine[1] == "-cc1") + return InitWithCommandLine(CommandLine); + + const auto MaybeFirstCC1 = getFirstCC1CommandLine( + ModifiedCommandLine, *DiagEngineWithCmdAndOpts->DiagEngine, OverlayFS); + if (!MaybeFirstCC1) + return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS); + return InitWithCommandLine(*MaybeFirstCC1); +} + +llvm::Expected +DependencyScanningTool::computeDependenciesByNameWithContextOrError( + StringRef ModuleName, const llvm::DenseSet &AlreadySeen, + LookupModuleOutputCallback LookupModuleOutput) { + FullDependencyConsumer Consumer(AlreadySeen); + CallbackActionController Controller(LookupModuleOutput); + if (Worker.computeDependenciesByNameWithContext(ModuleName, Consumer, + Controller)) + return Consumer.takeTranslationUnitDeps(); + return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS); +} + +llvm::Error +DependencyScanningTool::finalizeCompilerInstanceWithContextOrError() { + if (Worker.finalizeCompilerInstanceWithContext()) + return llvm::Error::success(); + return makeErrorFromDiagnosticsOS(*DiagPrinterWithOS); +} diff --git a/clang/test/ClangScanDeps/modules-full-by-mod-name.c b/clang/test/ClangScanDeps/modules-full-by-mod-name.c index edb99636aaf25..a2a2d3c22f9dd 100644 --- a/clang/test/ClangScanDeps/modules-full-by-mod-name.c +++ b/clang/test/ClangScanDeps/modules-full-by-mod-name.c @@ -17,17 +17,53 @@ module transitive { header "transitive.h" } // This is here to verify that the "root" directory doesn't clash with name of // the "root" module. +//--- main.cpp +// empty + //--- cdb.json.template [{ "file": "", "directory": "DIR", - "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c" + "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c main.cpp" }] // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s +//--- cdb.no-input.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.no-input.json.template > %t/cdb.no-input.json +// RUN: clang-scan-deps -compilation-database %t/cdb.no-input.json -format experimental-full -module-names=root > %t/result.no-input.json +// RUN: cat %t/result.no-input.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + +//--- cdb.cc1.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -I DIR -x c main.cpp" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.cc1.json.template > %t/cdb.cc1.json +// RUN: clang-scan-deps -compilation-database %t/cdb.cc1.json -format experimental-full -module-names=root > %t/result.cc1.json +// RUN: cat %t/result.cc1.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + +//--- cdb.cc1.no-input.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -I DIR -x c" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.cc1.no-input.json.template > %t/cdb.cc1.no-input.json +// RUN: clang-scan-deps -compilation-database %t/cdb.cc1.no-input.json -format experimental-full -module-names=root > %t/result.cc1.no-input.json +// RUN: cat %t/result.cc1.no-input.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { diff --git a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c index 030f7f3427810..3b7074219db84 100644 --- a/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c +++ b/clang/test/ClangScanDeps/modules-full-by-mult-mod-names.c @@ -22,17 +22,52 @@ module root1 { header "root1.h"} // This is here to verify that the "root" directory doesn't clash with name of // the "root" module. +//--- main.cpp + //--- cdb.json.template [{ "file": "", "directory": "DIR", - "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c" + "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c main.cpp" }] // RUN: sed "s|DIR|%/t|g" %t/cdb.json.template > %t/cdb.json // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -module-names=root,root1,direct > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s +//--- cdb.no-input.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -fmodules -fmodules-cache-path=DIR/cache -I DIR -x c" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.no-input.json.template > %t/cdb.no-input.json +// RUN: clang-scan-deps -compilation-database %t/cdb.no-input.json -format experimental-full -module-names=root,root1,direct > %t/result.no-input.json +// RUN: cat %t/result.no-input.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + +//--- cdb.cc1.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -I DIR -x c main.cpp" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.cc1.json.template > %t/cdb.cc1.json +// RUN: clang-scan-deps -compilation-database %t/cdb.cc1.json -format experimental-full -module-names=root,root1,direct > %t/result.cc1.json +// RUN: cat %t/result.cc1.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + +//--- cdb.cc1.no-input.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -I DIR -x c" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.cc1.no-input.json.template > %t/cdb.cc1.no-input.json +// RUN: clang-scan-deps -compilation-database %t/cdb.cc1.no-input.json -format experimental-full -module-names=root,root1,direct > %t/result.cc1.no-input.json +// RUN: cat %t/result.cc1.no-input.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s + // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { diff --git a/clang/test/ClangScanDeps/tu-buffer.c b/clang/test/ClangScanDeps/tu-buffer.c index b450b13ff434b..ce3b037174c26 100644 --- a/clang/test/ClangScanDeps/tu-buffer.c +++ b/clang/test/ClangScanDeps/tu-buffer.c @@ -37,6 +37,17 @@ module addition { header "addition.h" } // RUN: clang-scan-deps -compilation-database %t/cdb.json -format experimental-full -tu-buffer-path %t/tu.c > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s --check-prefix=CHECK +//--- cdb.cc1.json.template +[{ + "file": "", + "directory": "DIR", + "command": "clang -cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -I DIR -x c -emit-obj" +}] + +// RUN: sed "s|DIR|%/t|g" %t/cdb.cc1.json.template > %t/cdb.cc1.json +// RUN: clang-scan-deps -compilation-database %t/cdb.cc1.json -format experimental-full -tu-buffer-path %t/tu.c > %t/result.cc1.json +// RUN: cat %t/result.cc1.json | sed 's:\\\\\?:/:g' | FileCheck -DPREFIX=%/t %s --check-prefix=CHECK + // CHECK: { // CHECK-NEXT: "modules": [ // CHECK-NEXT: { diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp index 5f5bf42df5e6b..686b9e149ecc8 100644 --- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp +++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp @@ -6,14 +6,14 @@ // //===----------------------------------------------------------------------===// +#include "clang/DependencyScanning/DependencyScanningService.h" +#include "clang/DependencyScanning/DependencyScanningWorker.h" #include "clang/Driver/Compilation.h" #include "clang/Driver/Driver.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Tooling/CommonOptionsParser.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Tooling/DependencyScanningTool.h" #include "clang/Tooling/JSONCompilationDatabase.h" #include "clang/Tooling/Tooling.h" #include "llvm/ADT/STLExtras.h" @@ -40,7 +40,9 @@ #include "Opts.inc" using namespace clang; -using namespace tooling::dependencies; +using namespace tooling; +using namespace clang::dependencies; +using namespace clang::tooling::dependencies; namespace { @@ -1106,7 +1108,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { HadErrors = true; } else { if (llvm::Error Err = - WorkerTool.initializeCompilerInstanceWithContext( + WorkerTool.initializeCompilerInstanceWithContextOrError( CWD, Input->CommandLine)) { handleErrorWithInfoString( "Compiler instance with context setup error", std::move(Err), @@ -1117,7 +1119,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { for (auto N : Names) { auto MaybeModuleDepsGraph = - WorkerTool.computeDependenciesByNameWithContext( + WorkerTool.computeDependenciesByNameWithContextOrError( N, AlreadySeenModules, LookupOutput); if (handleModuleResult(N, MaybeModuleDepsGraph, *FD, LocalIndex, DependencyOS, Errs)) { @@ -1127,7 +1129,7 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) { } if (llvm::Error Err = - WorkerTool.finalizeCompilerInstanceWithContext()) { + WorkerTool.finalizeCompilerInstanceWithContextOrError()) { handleErrorWithInfoString( "Compiler instance with context finialization error", std::move(Err), DependencyOS, Errs); diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt index 54c781a35c20c..438a5c4c2e711 100644 --- a/clang/unittests/CMakeLists.txt +++ b/clang/unittests/CMakeLists.txt @@ -79,6 +79,7 @@ add_subdirectory(Basic) add_subdirectory(Lex) add_subdirectory(Parse) add_subdirectory(Driver) +add_subdirectory(DependencyScanning) if(CLANG_ENABLE_STATIC_ANALYZER) add_subdirectory(Analysis) add_subdirectory(StaticAnalyzer) diff --git a/clang/unittests/DependencyScanning/CMakeLists.txt b/clang/unittests/DependencyScanning/CMakeLists.txt new file mode 100644 index 0000000000000..40425820d4d08 --- /dev/null +++ b/clang/unittests/DependencyScanning/CMakeLists.txt @@ -0,0 +1,11 @@ +add_clang_unittest(ClangDependencyScanningTests + DependencyScanningFilesystemTest.cpp + DependencyScanningWorkerTest.cpp + CLANG_LIBS + clangDependencyScanning + clangFrontend # For TextDiagnosticPrinter. + LLVM_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + Option + Support + ) diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/DependencyScanning/DependencyScanningFilesystemTest.cpp similarity index 98% rename from clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp rename to clang/unittests/DependencyScanning/DependencyScanningFilesystemTest.cpp index cdb0ce2100d60..0e195411915aa 100644 --- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp +++ b/clang/unittests/DependencyScanning/DependencyScanningFilesystemTest.cpp @@ -6,12 +6,12 @@ // //===----------------------------------------------------------------------===// -#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h" +#include "clang/DependencyScanning/DependencyScanningFilesystem.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/VirtualFileSystem.h" #include "gtest/gtest.h" -using namespace clang::tooling::dependencies; +using namespace clang::dependencies; TEST(DependencyScanningFilesystem, OpenFileAndGetBufferRepeatedly) { auto InMemoryFS = llvm::makeIntrusiveRefCnt(); diff --git a/clang/unittests/DependencyScanning/DependencyScanningWorkerTest.cpp b/clang/unittests/DependencyScanning/DependencyScanningWorkerTest.cpp new file mode 100644 index 0000000000000..ca51d5ea982d9 --- /dev/null +++ b/clang/unittests/DependencyScanning/DependencyScanningWorkerTest.cpp @@ -0,0 +1,88 @@ +//===- DependencyScanningWorkerTest.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DependencyScanning/DependencyScanningWorker.h" +#include "clang/DependencyScanning/DependencyScanningUtils.h" +#include "llvm/Support/FormatVariadic.h" +#include "gtest/gtest.h" +#include + +using namespace clang; +using namespace dependencies; + +TEST(DependencyScanner, ScanDepsWithDiagConsumer) { + StringRef CWD = "/root"; + + auto VFS = llvm::makeIntrusiveRefCnt(); + VFS->setCurrentWorkingDirectory(CWD); + auto Sept = llvm::sys::path::get_separator(); + std::string HeaderPath = + std::string(llvm::formatv("{0}root{0}header.h", Sept)); + std::string TestPath = std::string(llvm::formatv("{0}root{0}test.cpp", Sept)); + std::string AsmPath = std::string(llvm::formatv("{0}root{0}test.s", Sept)); + + VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer("\n")); + VFS->addFile(TestPath, 0, + llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"\n")); + VFS->addFile(AsmPath, 0, llvm::MemoryBuffer::getMemBuffer("")); + + DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, + ScanningOutputFormat::Make); + DependencyScanningWorker Worker(Service, VFS); + + llvm::DenseSet AlreadySeen; + FullDependencyConsumer DC(AlreadySeen); + CallbackActionController AC(nullptr); + + struct EnsureFinishedConsumer : public DiagnosticConsumer { + bool Finished = false; + void finish() override { Finished = true; } + }; + + { + // Check that a successful scan calls DiagConsumer.finish(). + std::vector Args = { + "clang", "-cc1", "-triple", "x86_64-apple-macosx10.7.0", + "-emit-obj", "test.cpp", "-o", "test.cpp.o"}; + + EnsureFinishedConsumer DiagConsumer; + bool Success = Worker.computeDependencies(CWD, Args, DC, AC, DiagConsumer); + + EXPECT_TRUE(Success); + EXPECT_EQ(DiagConsumer.getNumErrors(), 0u); + EXPECT_TRUE(DiagConsumer.Finished); + } + + { + // Check that an invalid command-line, which never enters the scanning + // action calls DiagConsumer.finish(). + std::vector Args = {"clang", "-cc1", "-invalid-arg"}; + EnsureFinishedConsumer DiagConsumer; + bool Success = Worker.computeDependencies(CWD, Args, DC, AC, DiagConsumer); + + EXPECT_FALSE(Success); + EXPECT_GE(DiagConsumer.getNumErrors(), 1u); + EXPECT_TRUE(DiagConsumer.Finished); + } + + { + // Check that a valid command line that produces no scanning jobs calls + // DiagConsumer.finish(). + std::vector Args = { + "clang", "-cc1", "-triple", "x86_64-apple-macosx10.7.0", + "-emit-obj", "-x", "assembler", "test.s", + "-o", "test.cpp.o"}; + + EnsureFinishedConsumer DiagConsumer; + bool Success = Worker.computeDependencies(CWD, Args, DC, AC, DiagConsumer); + + EXPECT_FALSE(Success); + EXPECT_EQ(DiagConsumer.getNumErrors(), 1u); + EXPECT_TRUE(DiagConsumer.Finished); + } +} diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt index 106c6b9dc38bd..8c8b22250cd83 100644 --- a/clang/unittests/Tooling/CMakeLists.txt +++ b/clang/unittests/Tooling/CMakeLists.txt @@ -13,8 +13,7 @@ add_clang_unittest(ToolingTests LookupTest.cpp QualTypeNamesTest.cpp RangeSelectorTest.cpp - DependencyScanning/DependencyScannerTest.cpp - DependencyScanning/DependencyScanningFilesystemTest.cpp + DependencyScannerTest.cpp RecursiveASTVisitorTests/Attr.cpp RecursiveASTVisitorTests/BitfieldInitializer.cpp RecursiveASTVisitorTests/CallbacksLeaf.cpp diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScannerTest.cpp similarity index 78% rename from clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp rename to clang/unittests/Tooling/DependencyScannerTest.cpp index 4523af33e3c28..9fcd0545b17fa 100644 --- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp +++ b/clang/unittests/Tooling/DependencyScannerTest.cpp @@ -9,13 +9,13 @@ #include "clang/AST/ASTConsumer.h" #include "clang/AST/DeclCXX.h" #include "clang/AST/DeclGroup.h" +#include "clang/DependencyScanning/DependencyScanningWorker.h" #include "clang/Frontend/ASTUnit.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendAction.h" #include "clang/Frontend/FrontendActions.h" #include "clang/Tooling/CompilationDatabase.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" -#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h" +#include "clang/Tooling/DependencyScanningTool.h" #include "clang/Tooling/Tooling.h" #include "llvm/ADT/STLExtras.h" #include "llvm/MC/TargetRegistry.h" @@ -29,7 +29,8 @@ using namespace clang; using namespace tooling; -using namespace dependencies; +using namespace clang::dependencies; +using namespace tooling::dependencies; namespace { @@ -304,84 +305,3 @@ TEST(DependencyScanner, ScanDepsWithModuleLookup) { EXPECT_TRUE(!llvm::is_contained(InterceptFS->StatPaths, OtherPath)); EXPECT_EQ(InterceptFS->ReadFiles, std::vector{"test.m"}); } - -TEST(DependencyScanner, ScanDepsWithDiagConsumer) { - StringRef CWD = "/root"; - - auto VFS = llvm::makeIntrusiveRefCnt(); - VFS->setCurrentWorkingDirectory(CWD); - auto Sept = llvm::sys::path::get_separator(); - std::string HeaderPath = - std::string(llvm::formatv("{0}root{0}header.h", Sept)); - std::string TestPath = std::string(llvm::formatv("{0}root{0}test.cpp", Sept)); - std::string AsmPath = std::string(llvm::formatv("{0}root{0}test.s", Sept)); - - VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer("\n")); - VFS->addFile(TestPath, 0, - llvm::MemoryBuffer::getMemBuffer("#include \"header.h\"\n")); - VFS->addFile(AsmPath, 0, llvm::MemoryBuffer::getMemBuffer("")); - - DependencyScanningService Service(ScanningMode::DependencyDirectivesScan, - ScanningOutputFormat::Make); - DependencyScanningWorker Worker(Service, VFS); - - llvm::DenseSet AlreadySeen; - FullDependencyConsumer DC(AlreadySeen); - CallbackActionController AC(nullptr); - - struct EnsureFinishedConsumer : public DiagnosticConsumer { - bool Finished = false; - void finish() override { Finished = true; } - }; - - { - // Check that a successful scan calls DiagConsumer.finish(). - std::vector Args = {"clang", - "-target", - "x86_64-apple-macosx10.7", - "-c", - "test.cpp", - "-o" - "test.cpp.o"}; - - EnsureFinishedConsumer DiagConsumer; - bool Success = Worker.computeDependencies(CWD, Args, DC, AC, DiagConsumer); - - EXPECT_TRUE(Success); - EXPECT_EQ(DiagConsumer.getNumErrors(), 0u); - EXPECT_TRUE(DiagConsumer.Finished); - } - - { - // Check that an invalid command-line, which never enters the scanning - // action calls DiagConsumer.finish(). - std::vector Args = {"clang", "-invalid-arg"}; - EnsureFinishedConsumer DiagConsumer; - bool Success = Worker.computeDependencies(CWD, Args, DC, AC, DiagConsumer); - - EXPECT_FALSE(Success); - EXPECT_GE(DiagConsumer.getNumErrors(), 1u); - EXPECT_TRUE(DiagConsumer.Finished); - } - - { - // Check that a valid command line that produces no scanning jobs calls - // DiagConsumer.finish(). - std::vector Args = {"clang", - "-target", - "x86_64-apple-macosx10.7", - "-c", - "-x", - "assembler", - "test.s", - "-o" - "test.cpp.o"}; - - EnsureFinishedConsumer DiagConsumer; - bool Success = Worker.computeDependencies(CWD, Args, DC, AC, DiagConsumer); - - EXPECT_FALSE(Success); - EXPECT_EQ(DiagConsumer.getNumErrors(), 1u); - EXPECT_TRUE(DiagConsumer.Finished); - } -}