diff --git a/include/swift/DependencyScan/DependencyScanningTool.h b/include/swift/DependencyScan/DependencyScanningTool.h index 375a9deb6787f..1544109083252 100644 --- a/include/swift/DependencyScan/DependencyScanningTool.h +++ b/include/swift/DependencyScan/DependencyScanningTool.h @@ -15,20 +15,50 @@ #include "swift-c/DependencyScan/DependencyScan.h" #include "swift/Frontend/Frontend.h" +#include "swift/AST/DiagnosticConsumer.h" #include "swift/AST/ModuleDependencies.h" #include "swift/DependencyScan/ScanDependencies.h" #include "swift/Frontend/PrintingDiagnosticConsumer.h" +#include "swift/Frontend/SerializedDiagnosticConsumer.h" #include "llvm/Support/Error.h" #include "llvm/Support/StringSaver.h" namespace swift { namespace dependencies { class DependencyScanningTool; -class DependencyScanDiagnosticCollector; +class DepScanInMemoryDiagnosticCollector; -struct ScanQueryInstance { +struct ScanQueryContext { + /// Primary CompilerInstance configured for this scanning action std::unique_ptr ScanInstance; - std::shared_ptr ScanDiagnostics; + /// An thread-safe diagnostic consumer which collects all emitted + /// diagnostics in the scan to be reporte via libSwiftScan API + std::unique_ptr InMemoryDiagnosticCollector; + /// A thread-safe serialized diagnostics consumer. + /// Note, although type-erased, this must be an instance of + /// 'ThreadSafeSerializedDiagnosticConsumer' + std::unique_ptr SerializedDiagnosticConsumer; + + ScanQueryContext( + std::unique_ptr ScanInstance, + std::unique_ptr + InMemoryDiagnosticCollector, + std::unique_ptr SerializedDiagnosticConsumer) + : ScanInstance(std::move(ScanInstance)), + InMemoryDiagnosticCollector(std::move(InMemoryDiagnosticCollector)), + SerializedDiagnosticConsumer(std::move(SerializedDiagnosticConsumer)) {} + + ScanQueryContext(ScanQueryContext &&other) + : ScanInstance(std::move(other.ScanInstance)), + InMemoryDiagnosticCollector( + std::move(other.InMemoryDiagnosticCollector)), + SerializedDiagnosticConsumer( + std::move(other.SerializedDiagnosticConsumer)) {} + + ~ScanQueryContext() { + if (SerializedDiagnosticConsumer) + SerializedDiagnosticConsumer->finishProcessing(); + } }; /// Pure virtual Diagnostic consumer intended for collecting @@ -47,12 +77,16 @@ class ThreadSafeDiagnosticCollector : public DiagnosticConsumer { }; /// Diagnostic consumer that simply collects the diagnostics emitted so-far -class DependencyScanDiagnosticCollector : public ThreadSafeDiagnosticCollector { -private: +/// and uses a representation agnostic from any specific CompilerInstance state +/// which may have been used to emit the diagnostic +class DepScanInMemoryDiagnosticCollector + : public ThreadSafeDiagnosticCollector { +public: struct ScannerDiagnosticInfo { std::string Message; llvm::SourceMgr::DiagKind Severity; - std::optional ImportLocation; + std::optional + ImportLocation; }; std::vector Diagnostics; @@ -61,9 +95,12 @@ class DependencyScanDiagnosticCollector : public ThreadSafeDiagnosticCollector { public: friend DependencyScanningTool; - DependencyScanDiagnosticCollector() {} + DepScanInMemoryDiagnosticCollector() {} void reset() { Diagnostics.clear(); } - const std::vector &getDiagnostics() const { + std::vector getDiagnostics() const { + return Diagnostics; + } + const std::vector &getDiagnosticsRef() const { return Diagnostics; } }; @@ -97,10 +134,10 @@ class DependencyScanningTool { /// Using the specified invocation command, instantiate a CompilerInstance /// that will be used for this scan. - llvm::ErrorOr - initCompilerInstanceForScan(ArrayRef Command, - StringRef WorkingDirectory, - std::shared_ptr scannerDiagnosticsCollector); + llvm::ErrorOr createScanQueryContext( + ArrayRef Command, StringRef WorkingDirectory, + std::vector + &initializationDiagnostics); private: /// Shared cache of module dependencies, re-used by individual full-scan queries @@ -113,7 +150,9 @@ class DependencyScanningTool { llvm::StringSaver Saver; }; -swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput(const DependencyScanDiagnosticCollector *diagnosticCollector); +swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput( + ArrayRef + diagnostics); } // end namespace dependencies } // end namespace swift diff --git a/include/swift/DependencyScan/ScanDependencies.h b/include/swift/DependencyScan/ScanDependencies.h index f81757a7027c5..350680cafa2fb 100644 --- a/include/swift/DependencyScan/ScanDependencies.h +++ b/include/swift/DependencyScan/ScanDependencies.h @@ -43,7 +43,7 @@ using ModuleDependencyIDSet = class SwiftDependencyScanningService; namespace dependencies { -class DependencyScanDiagnosticCollector; +struct ScanQueryContext; using CompilerArgInstanceCacheMap = llvm::StringMap, @@ -62,16 +62,15 @@ bool prescanDependencies(CompilerInstance &instance); /// Scans the dependencies of the main module of \c instance. llvm::ErrorOr performModuleScan(SwiftDependencyScanningService &service, - CompilerInstance &instance, ModuleDependenciesCache &cache, - DependencyScanDiagnosticCollector *diagnostics = nullptr); + ScanQueryContext &queryContext); /// Scans the main module of \c instance for all direct module imports llvm::ErrorOr performModulePrescan(SwiftDependencyScanningService &service, - CompilerInstance &instance, ModuleDependenciesCache &cache, - DependencyScanDiagnosticCollector *diagnostics = nullptr); + ScanQueryContext &queryContext); + namespace incremental { /// For the given module dependency graph captured in the 'cache', diff --git a/include/swift/Frontend/SerializedDiagnosticConsumer.h b/include/swift/Frontend/SerializedDiagnosticConsumer.h index 3445ae4054e9a..689cba3827280 100644 --- a/include/swift/Frontend/SerializedDiagnosticConsumer.h +++ b/include/swift/Frontend/SerializedDiagnosticConsumer.h @@ -37,6 +37,15 @@ namespace swift { /// \returns A new diagnostic consumer that serializes diagnostics. std::unique_ptr createConsumer(llvm::StringRef outputPath, bool emitMacroExpansionFiles); + + /// Create a thread-safe DiagnosticConsumer that serializes diagnostics to a file, + /// using Clang's serialized diagnostics format. + /// + /// \param outputPath the file path to write the diagnostics to. + /// + /// \returns A new diagnostic consumer that serializes diagnostics. + std::unique_ptr + createThreadSafeConsumer(llvm::StringRef outputPath, bool emitMacroExpansionFiles); } } diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td index e95b32310dcc9..f084d3aad5605 100644 --- a/include/swift/Option/Options.td +++ b/include/swift/Option/Options.td @@ -2107,6 +2107,13 @@ def emit_module_serialize_diagnostics_path: "">, MetaVarName<"">; +def dependency_scan_serialize_diagnostics_path: + Separate<["-"], "dependency-scan-serialize-diagnostics-path">, + Flags<[ArgumentIsPath, SupplementaryOutput, NewDriverOnlyOption]>, + HelpText<"Emit a serialized diagnostics file for the dependency scanning task to " + "">, + MetaVarName<"">; + def emit_module_dependencies_path: Separate<["-"], "emit-module-dependencies-path">, Flags<[ArgumentIsPath, SupplementaryOutput, NewDriverOnlyOption]>, diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index cd8b67904563b..6a307bf40ff75 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -15,6 +15,7 @@ #include "swift/Basic/LLVMInitialize.h" #include "swift/Basic/TargetInfo.h" #include "swift/Basic/ColorUtils.h" +#include "swift/Basic/Defer.h" #include "swift/DependencyScan/DependencyScanningTool.h" #include "swift/DependencyScan/DependencyScanImpl.h" #include "swift/DependencyScan/SerializedModuleDependencyCacheFormat.h" @@ -73,7 +74,7 @@ void ThreadSafeDiagnosticCollector::handleDiagnostic(SourceManager &SM, } } -void DependencyScanDiagnosticCollector::addDiagnostic( +void DepScanInMemoryDiagnosticCollector::addDiagnostic( SourceManager &SM, const DiagnosticInfo &Info) { // Determine what kind of diagnostic we're emitting. llvm::SourceMgr::DiagKind SMKind; @@ -131,15 +132,15 @@ void DependencyScanDiagnosticCollector::addDiagnostic( } swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput( - const DependencyScanDiagnosticCollector *diagnosticCollector) { - auto collectedDiagnostics = diagnosticCollector->getDiagnostics(); - auto numDiagnostics = collectedDiagnostics.size(); + ArrayRef + diagnostics) { + auto numDiagnostics = diagnostics.size(); swiftscan_diagnostic_set_t *diagnosticOutput = new swiftscan_diagnostic_set_t; diagnosticOutput->count = numDiagnostics; diagnosticOutput->diagnostics = new swiftscan_diagnostic_info_t[numDiagnostics]; for (size_t i = 0; i < numDiagnostics; ++i) { - const auto &Diagnostic = collectedDiagnostics[i]; + const auto &Diagnostic = diagnostics[i]; swiftscan_diagnostic_info_s *diagnosticInfo = new swiftscan_diagnostic_info_s; diagnosticInfo->message = @@ -183,7 +184,8 @@ swiftscan_diagnostic_set_t *mapCollectedDiagnosticsForOutput( // module dependnecies but captures the diagnostics emitted during the attempted // scan query. static swiftscan_dependency_graph_t generateHollowDiagnosticOutput( - const DependencyScanDiagnosticCollector &ScanDiagnosticConsumer) { + ArrayRef + diagnostics) { // Create a dependency graph instance swiftscan_dependency_graph_t hollowResult = new swiftscan_dependency_graph_s; @@ -246,7 +248,7 @@ static swiftscan_dependency_graph_t generateHollowDiagnosticOutput( // Populate the diagnostic info hollowResult->diagnostics = - mapCollectedDiagnosticsForOutput(&ScanDiagnosticConsumer); + mapCollectedDiagnosticsForOutput(diagnostics); return hollowResult; } @@ -254,12 +256,13 @@ static swiftscan_dependency_graph_t generateHollowDiagnosticOutput( // imports but captures the diagnostics emitted during the attempted // scan query. static swiftscan_import_set_t generateHollowDiagnosticOutputImportSet( - const DependencyScanDiagnosticCollector &ScanDiagnosticConsumer) { + ArrayRef + diagnostics) { // Create an dependency graph instance swiftscan_import_set_t hollowResult = new swiftscan_import_set_s; hollowResult->imports = c_string_utils::create_empty_set(); hollowResult->diagnostics = - mapCollectedDiagnosticsForOutput(&ScanDiagnosticConsumer); + mapCollectedDiagnosticsForOutput(diagnostics); return hollowResult; } @@ -268,34 +271,31 @@ DependencyScanningTool::DependencyScanningTool() Alloc(), Saver(Alloc) {} llvm::ErrorOr -DependencyScanningTool::getDependencies( - ArrayRef Command, - StringRef WorkingDirectory) { - // There may be errors as early as in instance initialization, so we must ensure - // we can catch those. - auto ScanDiagnosticConsumer = - std::make_shared(); +DependencyScanningTool::getDependencies(ArrayRef Command, + StringRef WorkingDirectory) { + // Diagnostics which may get collected during scanning instance + // initialization + std::vector + InitializationDiagnostics; // The primary instance used to scan the query Swift source-code - auto QueryContextOrErr = initCompilerInstanceForScan(Command, - WorkingDirectory, - ScanDiagnosticConsumer); - if (QueryContextOrErr.getError()) - return generateHollowDiagnosticOutput(*ScanDiagnosticConsumer); - - auto QueryContext = std::move(*QueryContextOrErr); + auto QueryContext = createScanQueryContext(Command, WorkingDirectory, + InitializationDiagnostics); + if (QueryContext.getError()) + return generateHollowDiagnosticOutput(InitializationDiagnostics); + auto ScanInstance = QueryContext->ScanInstance.get(); // Local scan cache instance, wrapping the shared global cache. ModuleDependenciesCache cache( - QueryContext.ScanInstance->getMainModule()->getNameStr().str(), - QueryContext.ScanInstance->getInvocation().getModuleScanningHash()); + ScanInstance->getMainModule()->getNameStr().str(), + ScanInstance->getInvocation().getModuleScanningHash()); // Execute the scanning action, retrieving the in-memory result - auto DependenciesOrErr = performModuleScan(*ScanningService, - *QueryContext.ScanInstance.get(), - cache, - QueryContext.ScanDiagnostics.get()); + auto DependenciesOrErr = + performModuleScan(*ScanningService, cache, *QueryContext); + if (DependenciesOrErr.getError()) - return generateHollowDiagnosticOutput(*ScanDiagnosticConsumer); + return generateHollowDiagnosticOutput( + QueryContext->InMemoryDiagnosticCollector->getDiagnosticsRef()); return std::move(*DependenciesOrErr); } @@ -303,37 +303,37 @@ DependencyScanningTool::getDependencies( llvm::ErrorOr DependencyScanningTool::getImports(ArrayRef Command, StringRef WorkingDirectory) { - // There may be errors as early as in instance initialization, so we must ensure - // we can catch those - auto ScanDiagnosticConsumer = std::make_shared(); - // The primary instance used to scan the query Swift source-code - auto QueryContextOrErr = initCompilerInstanceForScan(Command, - WorkingDirectory, - ScanDiagnosticConsumer); - if (QueryContextOrErr.getError()) - return generateHollowDiagnosticOutputImportSet(*ScanDiagnosticConsumer); + // Diagnostics which may get collected during scanning instance + // initialization + std::vector + InitializationDiagnostics; - auto QueryContext = std::move(*QueryContextOrErr); + // The primary instance used to scan the query Swift source-code + auto QueryContext = createScanQueryContext(Command, WorkingDirectory, + InitializationDiagnostics); + if (QueryContext.getError()) + return generateHollowDiagnosticOutputImportSet(InitializationDiagnostics); + auto ScanInstance = QueryContext->ScanInstance.get(); // Local scan cache instance, wrapping the shared global cache. ModuleDependenciesCache cache( - QueryContext.ScanInstance->getMainModule()->getNameStr().str(), - QueryContext.ScanInstance->getInvocation().getModuleScanningHash()); - auto DependenciesOrErr = performModulePrescan(*ScanningService, - *QueryContext.ScanInstance.get(), - cache, - QueryContext.ScanDiagnostics.get()); + ScanInstance->getMainModule()->getNameStr().str(), + ScanInstance->getInvocation().getModuleScanningHash()); + // Execute the pre-scanning action, retrieving the in-memory result + auto DependenciesOrErr = + performModulePrescan(*ScanningService, cache, *QueryContext); + if (DependenciesOrErr.getError()) - return generateHollowDiagnosticOutputImportSet(*ScanDiagnosticConsumer); + return generateHollowDiagnosticOutputImportSet( + QueryContext->InMemoryDiagnosticCollector->getDiagnosticsRef()); return std::move(*DependenciesOrErr); } -llvm::ErrorOr -DependencyScanningTool::initCompilerInstanceForScan( - ArrayRef CommandArgs, - StringRef WorkingDir, - std::shared_ptr scannerDiagnosticsCollector) { +llvm::ErrorOr DependencyScanningTool::createScanQueryContext( + ArrayRef CommandArgs, StringRef WorkingDir, + std::vector + &InitializationDiagnostics) { // The remainder of this method operates on shared state in the // scanning service llvm::sys::SmartScopedLock Lock(DependencyScanningToolStateLock); @@ -342,57 +342,82 @@ DependencyScanningTool::initCompilerInstanceForScan( // client-side API plumbing. llvm::sys::SmartScopedLock TargetInfoLock(TargetInfoMutex); + // There may be errors as early as in instance initialization, so we must + // ensure we can catch those + auto ScannerDiagnosticsCollector = + std::make_unique(); + // State unique to an individual scan auto Instance = std::make_unique(); - Instance->addDiagnosticConsumer(scannerDiagnosticsCollector.get()); + Instance->addDiagnosticConsumer(ScannerDiagnosticsCollector.get()); - // Basic error checking on the arguments - if (CommandArgs.empty()) { - Instance->getDiags().diagnose(SourceLoc(), diag::error_no_frontend_args); - return std::make_error_code(std::errc::invalid_argument); - } + { + // In case we exit with an error, ensure the client gets the + // diagnostics collected so-far. + SWIFT_DEFER { + InitializationDiagnostics = ScannerDiagnosticsCollector->getDiagnostics(); + }; + + // Basic error checking on the arguments + if (CommandArgs.empty()) { + Instance->getDiags().diagnose(SourceLoc(), diag::error_no_frontend_args); + return std::make_error_code(std::errc::invalid_argument); + } - CompilerInvocation Invocation; - SmallString<128> WorkingDirectory(WorkingDir); - if (WorkingDirectory.empty()) - llvm::sys::fs::current_path(WorkingDirectory); + CompilerInvocation Invocation; + SmallString<128> WorkingDirectory(WorkingDir); + if (WorkingDirectory.empty()) + llvm::sys::fs::current_path(WorkingDirectory); - // Parse/tokenize arguments. - std::string CommandString; - for (const auto *c : CommandArgs) { - CommandString.append(c); - CommandString.append(" "); - } - SmallVector Args; - llvm::BumpPtrAllocator Alloc; - llvm::StringSaver Saver(Alloc); - // Ensure that we use the Windows command line parsing on Windows as we need - // to ensure that we properly handle paths. - if (llvm::Triple(llvm::sys::getProcessTriple()).isOSWindows()) - llvm::cl::TokenizeWindowsCommandLine(CommandString, Saver, Args); - else - llvm::cl::TokenizeGNUCommandLine(CommandString, Saver, Args); + // Parse/tokenize arguments. + std::string CommandString; + for (const auto *c : CommandArgs) { + CommandString.append(c); + CommandString.append(" "); + } + SmallVector Args; + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver(Alloc); + // Ensure that we use the Windows command line parsing on Windows as we need + // to ensure that we properly handle paths. + if (llvm::Triple(llvm::sys::getProcessTriple()).isOSWindows()) + llvm::cl::TokenizeWindowsCommandLine(CommandString, Saver, Args); + else + llvm::cl::TokenizeGNUCommandLine(CommandString, Saver, Args); + + if (Invocation.parseArgs(Args, Instance->getDiags(), + nullptr, WorkingDirectory, "/tmp/foo")) { + return std::make_error_code(std::errc::invalid_argument); + } - if (Invocation.parseArgs(Args, Instance->getDiags(), - nullptr, WorkingDirectory, "/tmp/foo")) { - return std::make_error_code(std::errc::invalid_argument); - } + // Setup the instance + std::string InstanceSetupError; + if (Instance->setup(Invocation, InstanceSetupError)) + return std::make_error_code(std::errc::not_supported); - // Setup the instance - std::string InstanceSetupError; - if (Instance->setup(Invocation, InstanceSetupError)) { - return std::make_error_code(std::errc::not_supported); - } - Invocation.getFrontendOptions().LLVMArgs.clear(); + Invocation.getFrontendOptions().LLVMArgs.clear(); - // Setup the caching service after the instance finishes setup. - if (ScanningService->setupCachingDependencyScanningService(*Instance)) - return std::make_error_code(std::errc::invalid_argument); + // Setup the caching service after the instance finishes setup. + if (ScanningService->setupCachingDependencyScanningService(*Instance)) + return std::make_error_code(std::errc::invalid_argument); + + (void)Instance->getMainModule(); + } - (void)Instance->getMainModule(); + auto SerializedDiagnosticsOutputPath = + Instance->getInvocation() + .getSerializedDiagnosticsPathForAtMostOnePrimary(); + std::unique_ptr SerailizedDiagnosticsConsumer; + if (!SerializedDiagnosticsOutputPath.empty()) { + SerailizedDiagnosticsConsumer = + swift::serialized_diagnostics::createThreadSafeConsumer( + SerializedDiagnosticsOutputPath, false); + Instance->addDiagnosticConsumer(SerailizedDiagnosticsConsumer.get()); + } - return ScanQueryInstance{std::move(Instance), - scannerDiagnosticsCollector}; + return ScanQueryContext{std::move(Instance), + std::move(ScannerDiagnosticsCollector), + std::move(SerailizedDiagnosticsConsumer)}; } } // namespace dependencies diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index e047089e53c4c..fc8853485f139 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -237,7 +237,7 @@ ModuleDependencyScanningWorker::ModuleDependencyScanningWorker( ScanASTContext.SourceMgr, *workerDiagnosticEngine)); scanningASTDelegate = std::make_unique( - workerASTContext->SourceMgr, &workerASTContext->Diags, + workerASTContext->SourceMgr, workerDiagnosticEngine.get(), workerASTContext->SearchPathOpts, workerASTContext->LangOpts, workerASTContext->ClangImporterOpts, workerASTContext->CASOpts, workerCompilerInvocation->getFrontendOptions(), @@ -314,14 +314,13 @@ ModuleDependencyScanningWorker::scanFilesystemForClangModuleDependency( clangScanningWorkingDirectoryPath, alreadySeenModules, lookupModuleOutput); if (!clangModuleDependencies) { - auto errorStr = toString(clangModuleDependencies.takeError()); - // We ignore the "module 'foo' not found" error, the Swift dependency - // scanner will report such an error only if all of the module loaders - // fail as well. - if (errorStr.find("fatal error: module '" + moduleName.str().str() + - "' not found") == std::string::npos) - workerASTContext->Diags.diagnose( - SourceLoc(), diag::clang_dependency_scan_error, errorStr); + llvm::handleAllErrors(clangModuleDependencies.takeError(), [this, &moduleName]( + const llvm::StringError &E) { + auto &message = E.getMessage(); + if (message.find("fatal error: module '" + moduleName.str().str() + + "' not found") == std::string::npos) + workerDiagnosticEngine->diagnose(SourceLoc(), diag::clang_dependency_scan_error, message); + }); return std::nullopt; } @@ -355,7 +354,7 @@ ModuleDependencyScanningWorker::scanHeaderDependenciesOfSwiftModule( auto clangModuleDependencies = scanHeaderDependencies(); if (!clangModuleDependencies) { auto errorStr = toString(clangModuleDependencies.takeError()); - workerASTContext->Diags.diagnose( + workerDiagnosticEngine->diagnose( SourceLoc(), diag::clang_header_dependency_scan_error, errorStr); return std::nullopt; } diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index e5e3d29add55c..b19ade93112a7 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -44,6 +44,7 @@ #include "swift/Frontend/Frontend.h" #include "swift/Frontend/FrontendOptions.h" #include "swift/Frontend/ModuleInterfaceLoader.h" +#include "swift/Frontend/SerializedDiagnosticConsumer.h" #include "swift/Strings.h" #include "clang/CAS/IncludeTree.h" #include "llvm/ADT/STLExtras.h" @@ -737,7 +738,7 @@ static swiftscan_macro_dependency_set_t *createMacroDependencySet( static swiftscan_dependency_graph_t generateFullDependencyGraph( const CompilerInstance &instance, - const DependencyScanDiagnosticCollector *diagnosticCollector, + const DepScanInMemoryDiagnosticCollector *diagnosticCollector, const ModuleDependenciesCache &cache, const ArrayRef allModules) { if (allModules.empty()) { @@ -965,10 +966,10 @@ static swiftscan_dependency_graph_t generateFullDependencyGraph( swiftscan_dependency_graph_t result = new swiftscan_dependency_graph_s; result->main_module_name = create_clone(mainModuleName.c_str()); result->dependencies = dependencySet; - result->diagnostics = - diagnosticCollector - ? mapCollectedDiagnosticsForOutput(diagnosticCollector) - : nullptr; + result->diagnostics = diagnosticCollector + ? mapCollectedDiagnosticsForOutput( + diagnosticCollector->getDiagnostics()) + : nullptr; return result; } @@ -1246,90 +1247,6 @@ static bool diagnoseCycle(const CompilerInstance &instance, closeSet.clear(); return false; } -} // namespace - -bool swift::dependencies::scanDependencies(CompilerInstance &CI) { - ASTContext &ctx = CI.getASTContext(); - std::string depGraphOutputPath = - CI.getInvocation() - .getFrontendOptions() - .InputsAndOutputs.getSingleOutputFilename(); - // `-scan-dependencies` invocations use a single new instance - // of a module cache - SwiftDependencyScanningService *service = - ctx.Allocate(); - ModuleDependenciesCache cache(CI.getMainModule()->getNameStr().str(), - CI.getInvocation().getModuleScanningHash()); - - if (service->setupCachingDependencyScanningService(CI)) - return true; - - // Execute scan - llvm::ErrorOr dependenciesOrErr = - performModuleScan(*service, CI, cache); - - if (dependenciesOrErr.getError()) - return true; - auto dependencies = std::move(*dependenciesOrErr); - - if (writeJSONToOutput(ctx.Diags, CI.getOutputBackend(), depGraphOutputPath, - dependencies)) - return true; - - // This process succeeds regardless of whether any errors occurred. - // FIXME: We shouldn't need this, but it's masking bugs in our scanning - // logic where we don't create a fresh context when scanning Swift interfaces - // that includes their own command-line flags. - ctx.Diags.resetHadAnyError(); - return false; -} - -bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { - ASTContext &Context = instance.getASTContext(); - const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); - std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); - // `-scan-dependencies` invocations use a single new instance - // of a module cache - SwiftDependencyScanningService *singleUseService = - Context.Allocate(); - ModuleDependenciesCache cache( - instance.getMainModule()->getNameStr().str(), - instance.getInvocation().getModuleScanningHash()); - - // Execute import prescan, and write JSON output to the output stream - auto importSetOrErr = - performModulePrescan(*singleUseService, instance, cache); - if (importSetOrErr.getError()) - return true; - auto importSet = std::move(*importSetOrErr); - - // Serialize and output main module dependencies only and exit. - if (writePrescanJSONToOutput(Context.Diags, instance.getOutputBackend(), path, - importSet)) - return true; - - // This process succeeds regardless of whether any errors occurred. - // FIXME: We shouldn't need this, but it's masking bugs in our scanning - // logic where we don't create a fresh context when scanning Swift interfaces - // that includes their own command-line flags. - Context.Diags.resetHadAnyError(); - return false; -} - -std::string -swift::dependencies::createEncodedModuleKindAndName(ModuleDependencyID id) { - switch (id.Kind) { - case ModuleDependencyKind::SwiftInterface: - case ModuleDependencyKind::SwiftSource: - return "swiftTextual:" + id.ModuleName; - case ModuleDependencyKind::SwiftBinary: - return "swiftBinary:" + id.ModuleName; - case ModuleDependencyKind::Clang: - return "clang:" + id.ModuleName; - default: - llvm_unreachable("Unhandled dependency kind."); - } -} static bool resolveDependencyCommandLineArguments( ModuleDependencyScanner &scanner, CompilerInstance &instance, @@ -1429,13 +1346,13 @@ static void resolveImplicitLinkLibraries(const CompilerInstance &instance, cache.updateDependency(mainModuleID, mainModuleDepInfo); } -llvm::ErrorOr -swift::dependencies::performModuleScan( - SwiftDependencyScanningService &service, CompilerInstance &instance, +static llvm::ErrorOr +performModuleScanImpl( + SwiftDependencyScanningService &service, CompilerInstance *instance, ModuleDependenciesCache &cache, - DependencyScanDiagnosticCollector *diagnosticCollector) { - const ASTContext &ctx = instance.getASTContext(); - const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); + DepScanInMemoryDiagnosticCollector *diagnosticCollector) { + const ASTContext &ctx = instance->getASTContext(); + const FrontendOptions &opts = instance->getInvocation().getFrontendOptions(); // Load the dependency cache if -reuse-dependency-scan-cache // is specified if (opts.ReuseDependencyScannerCache) { @@ -1453,43 +1370,43 @@ swift::dependencies::performModuleScan( if (!loadFailure && opts.ValidatePriorDependencyScannerCache) { auto mainModuleID = - ModuleDependencyID{instance.getMainModule()->getNameStr().str(), + ModuleDependencyID{instance->getMainModule()->getNameStr().str(), ModuleDependencyKind::SwiftSource}; incremental::validateInterModuleDependenciesCache( - mainModuleID, cache, instance.getSharedCASInstance(), - serializedCacheTimeStamp, *instance.getSourceMgr().getFileSystem(), + mainModuleID, cache, instance->getSharedCASInstance(), + serializedCacheTimeStamp, *instance->getSourceMgr().getFileSystem(), ctx.Diags, opts.EmitDependencyScannerCacheRemarks); } } auto scanner = ModuleDependencyScanner( - service, instance.getInvocation(), instance.getSILOptions(), - instance.getASTContext(), *instance.getDependencyTracker(), - instance.getSharedCASInstance(), instance.getSharedCacheInstance(), - instance.getDiags(), - instance.getInvocation().getFrontendOptions().ParallelDependencyScan); + service, instance->getInvocation(), instance->getSILOptions(), + instance->getASTContext(), *instance->getDependencyTracker(), + instance->getSharedCASInstance(), instance->getSharedCacheInstance(), + instance->getDiags(), + instance->getInvocation().getFrontendOptions().ParallelDependencyScan); // Identify imports of the main module and add an entry for it // to the dependency graph. - auto mainModuleName = instance.getMainModule()->getNameStr(); + auto mainModuleName = instance->getMainModule()->getNameStr(); auto mainModuleID = ModuleDependencyID{mainModuleName.str(), ModuleDependencyKind::SwiftSource}; if (!cache.hasDependency(mainModuleID)) cache.recordDependency(mainModuleName, *scanner.getMainModuleDependencyInfo( - instance.getMainModule())); + instance->getMainModule())); // Perform the full module scan starting at the main module. auto allModules = scanner.performDependencyScan(mainModuleID, cache); - if (diagnoseCycle(instance, cache, mainModuleID)) + if (diagnoseCycle(*instance, cache, mainModuleID)) return std::make_error_code(std::errc::not_supported); auto topologicallySortedModuleList = computeTopologicalSortOfExplicitDependencies(allModules, cache); - resolveDependencyCommandLineArguments(scanner, instance, cache, + resolveDependencyCommandLineArguments(scanner, *instance, cache, topologicallySortedModuleList); - resolveImplicitLinkLibraries(instance, cache); - updateDependencyTracker(instance, cache, allModules); + resolveImplicitLinkLibraries(*instance, cache); + updateDependencyTracker(*instance, cache, allModules); if (ctx.Stats) ctx.Stats->getFrontendCounters().NumDepScanFilesystemLookups = @@ -1500,29 +1417,29 @@ swift::dependencies::performModuleScan( if (opts.SerializeDependencyScannerCache) { auto savePath = opts.SerializedDependencyScannerCachePath; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - ctx.Diags, instance.getOutputBackend(), savePath, cache); + ctx.Diags, instance->getOutputBackend(), savePath, cache); if (opts.EmitDependencyScannerCacheRemarks) ctx.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath); } - return generateFullDependencyGraph(instance, diagnosticCollector, cache, - topologicallySortedModuleList); + return generateFullDependencyGraph(*instance, diagnosticCollector, + cache, topologicallySortedModuleList); } -llvm::ErrorOr swift::dependencies::performModulePrescan( - SwiftDependencyScanningService &service, CompilerInstance &instance, +static llvm::ErrorOr performModulePrescanImpl( + SwiftDependencyScanningService &service, CompilerInstance *instance, ModuleDependenciesCache &cache, - DependencyScanDiagnosticCollector *diagnosticCollector) { + DepScanInMemoryDiagnosticCollector *diagnosticCollector) { // Setup the scanner auto scanner = ModuleDependencyScanner( - service, instance.getInvocation(), instance.getSILOptions(), - instance.getASTContext(), *instance.getDependencyTracker(), - instance.getSharedCASInstance(), instance.getSharedCacheInstance(), - instance.getDiags(), - instance.getInvocation().getFrontendOptions().ParallelDependencyScan); + service, instance->getInvocation(), instance->getSILOptions(), + instance->getASTContext(), *instance->getDependencyTracker(), + instance->getSharedCASInstance(), instance->getSharedCacheInstance(), + instance->getDiags(), + instance->getInvocation().getFrontendOptions().ParallelDependencyScan); // Execute import prescan, and write JSON output to the output stream auto mainDependencies = - scanner.getMainModuleDependencyInfo(instance.getMainModule()); + scanner.getMainModuleDependencyInfo(instance->getMainModule()); if (!mainDependencies) return mainDependencies.getError(); auto *importSet = new swiftscan_import_set_s; @@ -1535,16 +1452,116 @@ llvm::ErrorOr swift::dependencies::performModulePrescan( return importInfo.importIdentifier; }); importSet->imports = create_set(importIdentifiers); - importSet->diagnostics = - diagnosticCollector - ? mapCollectedDiagnosticsForOutput(diagnosticCollector) - : nullptr; - importSet->diagnostics = - diagnosticCollector - ? mapCollectedDiagnosticsForOutput(diagnosticCollector) - : nullptr; + importSet->diagnostics = diagnosticCollector + ? mapCollectedDiagnosticsForOutput( + diagnosticCollector->getDiagnostics()) + : nullptr; + importSet->diagnostics = diagnosticCollector + ? mapCollectedDiagnosticsForOutput( + diagnosticCollector->getDiagnostics()) + : nullptr; return importSet; } +} // namespace + +bool swift::dependencies::scanDependencies(CompilerInstance &CI) { + ASTContext &ctx = CI.getASTContext(); + std::string depGraphOutputPath = + CI.getInvocation() + .getFrontendOptions() + .InputsAndOutputs.getSingleOutputFilename(); + // `-scan-dependencies` invocations use a single new instance + // of a module cache + SwiftDependencyScanningService *service = + ctx.Allocate(); + ModuleDependenciesCache cache(CI.getMainModule()->getNameStr().str(), + CI.getInvocation().getModuleScanningHash()); + + if (service->setupCachingDependencyScanningService(CI)) + return true; + + // Execute scan + llvm::ErrorOr dependenciesOrErr = + performModuleScanImpl(*service, &CI, cache, nullptr); + + if (dependenciesOrErr.getError()) + return true; + auto dependencies = std::move(*dependenciesOrErr); + + if (writeJSONToOutput(ctx.Diags, CI.getOutputBackend(), depGraphOutputPath, + dependencies)) + return true; + + // This process succeeds regardless of whether any errors occurred. + // FIXME: We shouldn't need this, but it's masking bugs in our scanning + // logic where we don't create a fresh context when scanning Swift interfaces + // that includes their own command-line flags. + ctx.Diags.resetHadAnyError(); + return false; +} + +bool swift::dependencies::prescanDependencies(CompilerInstance &instance) { + ASTContext &Context = instance.getASTContext(); + const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); + std::string path = opts.InputsAndOutputs.getSingleOutputFilename(); + // `-scan-dependencies` invocations use a single new instance + // of a module cache + SwiftDependencyScanningService *singleUseService = + Context.Allocate(); + ModuleDependenciesCache cache( + instance.getMainModule()->getNameStr().str(), + instance.getInvocation().getModuleScanningHash()); + + // Execute import prescan, and write JSON output to the output stream + auto importSetOrErr = + performModulePrescanImpl(*singleUseService, &instance, cache, nullptr); + if (importSetOrErr.getError()) + return true; + auto importSet = std::move(*importSetOrErr); + + // Serialize and output main module dependencies only and exit. + if (writePrescanJSONToOutput(Context.Diags, instance.getOutputBackend(), path, + importSet)) + return true; + + // This process succeeds regardless of whether any errors occurred. + // FIXME: We shouldn't need this, but it's masking bugs in our scanning + // logic where we don't create a fresh context when scanning Swift interfaces + // that includes their own command-line flags. + Context.Diags.resetHadAnyError(); + return false; +} + +std::string +swift::dependencies::createEncodedModuleKindAndName(ModuleDependencyID id) { + switch (id.Kind) { + case ModuleDependencyKind::SwiftInterface: + case ModuleDependencyKind::SwiftSource: + return "swiftTextual:" + id.ModuleName; + case ModuleDependencyKind::SwiftBinary: + return "swiftBinary:" + id.ModuleName; + case ModuleDependencyKind::Clang: + return "clang:" + id.ModuleName; + default: + llvm_unreachable("Unhandled dependency kind."); + } +} + +llvm::ErrorOr +swift::dependencies::performModuleScan(SwiftDependencyScanningService &service, + ModuleDependenciesCache &cache, + ScanQueryContext &queryContext) { + return performModuleScanImpl(service, queryContext.ScanInstance.get(), cache, + queryContext.InMemoryDiagnosticCollector.get()); +} + +llvm::ErrorOr swift::dependencies::performModulePrescan( + SwiftDependencyScanningService &service, ModuleDependenciesCache &cache, + ScanQueryContext &queryContext) { + return performModulePrescanImpl( + service, queryContext.ScanInstance.get(), cache, + queryContext.InMemoryDiagnosticCollector.get()); +} void swift::dependencies::incremental::validateInterModuleDependenciesCache( const ModuleDependencyID &rootModuleID, ModuleDependenciesCache &cache, diff --git a/lib/Frontend/SerializedDiagnosticConsumer.cpp b/lib/Frontend/SerializedDiagnosticConsumer.cpp index 3436805b6b65b..8818a778c4aa2 100644 --- a/lib/Frontend/SerializedDiagnosticConsumer.cpp +++ b/lib/Frontend/SerializedDiagnosticConsumer.cpp @@ -32,6 +32,7 @@ #include "llvm/ADT/IntrusiveRefCntPtr.h" #include "llvm/ADT/SmallString.h" #include "llvm/Bitstream/BitstreamWriter.h" +#include "llvm/Support/Mutex.h" using namespace swift; using namespace clang::serialized_diags; @@ -217,6 +218,39 @@ class SerializedDiagnosticConsumer : public DiagnosticConsumer { DiagnosticKind Kind, StringRef Text, const DiagnosticInfo &Info); }; + +/// A thread-safe version of SerializedDiagnosticConsumer which +/// serializes access to all public API of the consumer. +class ThreadSafeSerializedDiagnosticConsumer + : public SerializedDiagnosticConsumer { +private: + llvm::sys::SmartMutex DiagnosticConsumerStateLock; + +public: + ThreadSafeSerializedDiagnosticConsumer(StringRef serializedDiagnosticsPath, + bool emitMacroExpansionFiles) + : SerializedDiagnosticConsumer(serializedDiagnosticsPath, + emitMacroExpansionFiles) {} + + bool finishProcessing() override { + llvm::sys::SmartScopedLock Lock(DiagnosticConsumerStateLock); + // Note: this is only synchronized behind a log to ease debugging possible + // issues, this method must not be called more than once. + return SerializedDiagnosticConsumer::finishProcessing(); + } + + void informDriverOfIncompleteBatchModeCompilation() override { + llvm::sys::SmartScopedLock Lock(DiagnosticConsumerStateLock); + SerializedDiagnosticConsumer:: + informDriverOfIncompleteBatchModeCompilation(); + } + + void handleDiagnostic(SourceManager &SM, + const DiagnosticInfo &Info) override { + llvm::sys::SmartScopedLock Lock(DiagnosticConsumerStateLock); + SerializedDiagnosticConsumer::handleDiagnostic(SM, Info); + } +}; } // end anonymous namespace namespace swift { @@ -227,6 +261,13 @@ namespace serialized_diagnostics { return std::make_unique( outputPath, emitMacroExpansionFiles); } + + std::unique_ptr createThreadSafeConsumer( + StringRef outputPath, bool emitMacroExpansionFiles + ) { + return std::make_unique( + outputPath, emitMacroExpansionFiles); + } } // namespace serialized_diagnostics } // namespace swift diff --git a/unittests/DependencyScan/ModuleDeps.cpp b/unittests/DependencyScan/ModuleDeps.cpp index 92e97f01c9319..e932fd8bf937a 100644 --- a/unittests/DependencyScan/ModuleDeps.cpp +++ b/unittests/DependencyScan/ModuleDeps.cpp @@ -14,6 +14,8 @@ #include "swift/Basic/Defer.h" #include "swift/Basic/Platform.h" #include "swift/DependencyScan/DependencyScanImpl.h" +#include "clang/Frontend/SerializedDiagnosticReader.h" +#include "clang/Frontend/SerializedDiagnostics.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TargetParser/Host.h" @@ -244,12 +246,125 @@ public func overlayFuncA() { }\n")); CommandB.push_back(command.c_str()); } - auto ScanDiagnosticConsumer = std::make_shared(); - auto instanceA = ScannerTool.initCompilerInstanceForScan(CommandA, {}, ScanDiagnosticConsumer); - auto instanceB = ScannerTool.initCompilerInstanceForScan(CommandB, {}, ScanDiagnosticConsumer); + std::vector + InitializationDiagnostics; + auto queryAContext = ScannerTool.createScanQueryContext(CommandA, {}, + InitializationDiagnostics); + auto queryBContext = ScannerTool.createScanQueryContext(CommandB, {}, + InitializationDiagnostics); // Ensure that scans that only differ in module name have distinct scanning context hashes - ASSERT_NE(instanceA->ScanInstance.get()->getInvocation().getModuleScanningHash(), - instanceB->ScanInstance.get()->getInvocation().getModuleScanningHash()); + ASSERT_NE(queryAContext->ScanInstance.get()->getInvocation().getModuleScanningHash(), + queryBContext->ScanInstance.get()->getInvocation().getModuleScanningHash()); +} + +namespace { +class DiagnosticChecker : public clang::serialized_diags::SerializedDiagnosticReader { +public: + std::vector errorMessages; + std::vector warningMessages; + +protected: + std::error_code visitDiagnosticRecord( + unsigned Severity, const clang::serialized_diags::Location &Location, + unsigned Category, unsigned Flag, StringRef Message) override { + switch (static_cast(Severity)) { + case clang::serialized_diags::Warning: + warningMessages.push_back(Message.str()); + break; + case clang::serialized_diags::Error: + errorMessages.push_back(Message.str()); + break; + default: + break; + } + return std::error_code(); + } +}; +} + + +TEST_F(ScanTest, TestSerializedDiagnosticOutput) { + SmallString<256> tempDir; + ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory( + "ScanTest.TestSerializedDiagnosticOutput", tempDir)); + SWIFT_DEFER { llvm::sys::fs::remove_directories(tempDir); }; + + // Create test input file + std::string TestPathStr = createFilename(tempDir, "foo.swift"); + ASSERT_FALSE(emitFileWithContents(tempDir, "foo.swift", "import A\n\ +#warning(\"This is a warning\")\n")); + + // Create include directory + std::string SwiftDirPath = createFilename(tempDir, "Swift"); + ASSERT_FALSE(llvm::sys::fs::create_directory(SwiftDirPath)); + + // Create output directory + std::string OutputDirPath = createFilename(tempDir, "Output"); + ASSERT_FALSE(llvm::sys::fs::create_directory(OutputDirPath)); + std::string SerializedDiagnosticsOutputPath = + createFilename(OutputDirPath, "scan-diags.dia"); + + // Create imported module Swift interface files + ASSERT_FALSE(emitFileWithContents(SwiftDirPath, "A.swiftinterface", + "// swift-interface-format-version: 1.0\n\ +// swift-module-flags: -module-name A\n\ +import Swift\n\ +#error(\"This is an error\")\n\ +public func funcA() { }\n")); + + // Paths to shims and stdlib + llvm::SmallString<128> ShimsLibDir = StdLibDir; + llvm::sys::path::append(ShimsLibDir, "shims"); + auto Target = llvm::Triple(llvm::sys::getDefaultTargetTriple()); + llvm::sys::path::append(StdLibDir, getPlatformNameForTriple(Target)); + + // Generate command line + std::vector CommandStr = { + TestPathStr, + std::string("-I ") + SwiftDirPath, + std::string("-I ") + StdLibDir.str().str(), + std::string("-I ") + ShimsLibDir.str().str(), + "-serialize-diagnostics-path", + SerializedDiagnosticsOutputPath, + "-module-name", + "testSerializedDiagnosticOutput"}; + // On Windows we need to add an extra escape for path separator characters + // because otherwise the command line tokenizer will treat them as escape + // characters. + for (size_t i = 0; i < CommandStr.size(); ++i) { + std::replace(CommandStr[i].begin(), CommandStr[i].end(), '\\', '/'); + } + std::vector Command; + for (auto &command : CommandStr) + Command.push_back(command.c_str()); + + { + auto ScanningService = std::make_unique(); + std::vector + InitializationDiagnostics; + auto QueryContext = ScannerTool.createScanQueryContext(Command, {}, + InitializationDiagnostics); + ASSERT_FALSE(QueryContext.getError()); + + ModuleDependenciesCache ScanCache( + QueryContext->ScanInstance.get()->getMainModule()->getNameStr().str(), + QueryContext->ScanInstance.get() + ->getInvocation() + .getModuleScanningHash()); + auto DependenciesOrErr = + performModuleScan(*ScanningService, ScanCache, *QueryContext); + + ASSERT_FALSE(DependenciesOrErr.getError()); + } + ASSERT_TRUE(llvm::sys::fs::exists(SerializedDiagnosticsOutputPath)); + + auto DiagnosticsReader = DiagnosticChecker(); + auto ReadError = DiagnosticsReader.readDiagnostics(SerializedDiagnosticsOutputPath); + ASSERT_FALSE(ReadError); + ASSERT_TRUE(DiagnosticsReader.errorMessages.size() == 1); + ASSERT_TRUE(DiagnosticsReader.warningMessages.size() == 1); + ASSERT_TRUE(DiagnosticsReader.errorMessages.front() == "This is an error"); + ASSERT_TRUE(DiagnosticsReader.warningMessages.front() == "This is a warning"); } TEST_F(ScanTest, TestModuleCycle) {