diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index 9ef666affbf7e..9281ebcf55c2b 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -739,6 +739,12 @@ class ASTContext final { ModuleDependenciesCache &cache, InterfaceSubContextDelegate &delegate); + /// Retrieve the module dependencies for the Swift module with the given name. + Optional getSwiftModuleDependencies( + StringRef moduleName, + ModuleDependenciesCache &cache, + InterfaceSubContextDelegate &delegate); + /// Load extensions to the given nominal type from the external /// module loaders. /// diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index 21931481da7be..36471ecbc9d92 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -274,6 +274,14 @@ ERROR(placeholder_dependency_module_map_corrupted,none, "Swift placeholder dependency module map from %0 is malformed", (StringRef)) +ERROR(batch_scan_input_file_missing,none, + "cannot open batch dependencies scan input file from %0", + (StringRef)) + +ERROR(batch_scan_input_file_corrupted,none, + "batch dependencies scan input file from %0 is malformed", + (StringRef)) + REMARK(default_previous_install_name, none, "default previous install name for %0 is %1", (StringRef, StringRef)) diff --git a/include/swift/AST/SearchPathOptions.h b/include/swift/AST/SearchPathOptions.h index 08757ffd796de..52eeace6a1460 100644 --- a/include/swift/AST/SearchPathOptions.h +++ b/include/swift/AST/SearchPathOptions.h @@ -93,6 +93,9 @@ class SearchPathOptions { /// A map of placeholder Swift module dependency information. std::string PlaceholderDependencyModuleMap; + + /// A file containing modules we should perform batch scanning. + std::string BatchScanInputFilePath; private: static StringRef pathStringFromFrameworkSearchPath(const FrameworkSearchPath &next) { diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 5bd693d2ffe7d..8c00c8a94ca16 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -228,6 +228,10 @@ def explict_swift_module_map def placeholder_dependency_module_map : Separate<["-"], "placeholder-dependency-module-map-file">, MetaVarName<"">, HelpText<"Specify a JSON file containing information of external Swift module dependencies">; + +def batch_scan_input_file + : Separate<["-"], "batch-scan-input-file">, MetaVarName<"">, + HelpText<"Specify a JSON file containing modules to perform batch dependencies scanning">; } diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index e75c5185cc13b..f3ee9197874a9 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -1516,6 +1516,21 @@ Optional ASTContext::getModuleDependencies( return None; } +Optional +ASTContext::getSwiftModuleDependencies(StringRef moduleName, + ModuleDependenciesCache &cache, + InterfaceSubContextDelegate &delegate) { + for (auto &loader : getImpl().ModuleLoaders) { + if (loader.get() == getImpl().TheClangModuleLoader) + continue; + + if (auto dependencies = loader->getModuleDependencies(moduleName, cache, + delegate)) + return dependencies; + } + return None; +} + void ASTContext::loadExtensions(NominalTypeDecl *nominal, unsigned previousGeneration) { PrettyStackTraceDecl stackTrace("loading extensions for", nominal); diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index f9c3828c19c67..db5894b1d4a20 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -926,7 +926,8 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, } if (const Arg *A = Args.getLastArg(OPT_placeholder_dependency_module_map)) Opts.PlaceholderDependencyModuleMap = A->getValue(); - + if (const Arg *A = Args.getLastArg(OPT_batch_scan_input_file)) + Opts.BatchScanInputFilePath = A->getValue(); // Opts.RuntimeIncludePath is set by calls to // setRuntimeIncludePath() or setMainExecutablePath(). // Opts.RuntimeImportPath is set by calls to diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 9bdfe3e1da24b..063e89e2860d6 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -1833,8 +1833,14 @@ static bool performCompile(CompilerInstance &Instance, return finishPipeline(Context.hadError()); } - if (Action == FrontendOptions::ActionType::ScanDependencies) - return finishPipeline(scanDependencies(Instance)); + if (Action == FrontendOptions::ActionType::ScanDependencies) { + auto batchScanInput = Instance.getASTContext().SearchPathOpts.BatchScanInputFilePath; + if (batchScanInput.empty()) + return finishPipeline(scanDependencies(Instance)); + else + return finishPipeline(batchScanModuleDependencies(Instance, + batchScanInput)); + } if (Action == FrontendOptions::ActionType::ScanClangDependencies) return finishPipeline(scanClangDependencies(Instance)); diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index bf2fccc25de93..524a5de2213e6 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -32,12 +32,103 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/YAMLParser.h" #include using namespace swift; +using namespace llvm::yaml; namespace { +struct BatchScanInput { + StringRef moduleName; + StringRef arguments; + StringRef outputPath; + bool isSwift; +}; + +static std::string getScalaNodeText(Node *N) { + SmallString<32> Buffer; + return cast(N)->getValue(Buffer).str(); +} + +/// Parse an entry like this, where the "platforms" key-value pair is optional: +/// { +/// "swiftModuleName": "Foo", +/// "arguments": "-target 10.15", +/// "output": "../Foo.json" +/// }, +static bool parseBatchInputEntries(ASTContext &Ctx, llvm::StringSaver &saver, + Node *Node, std::vector &result) { + auto *SN = cast(Node); + if (!SN) + return true; + for (auto It = SN->begin(); It != SN->end(); ++It) { + auto *MN = cast(&*It); + BatchScanInput entry; + Optional> Platforms; + for (auto &Pair: *MN) { + auto Key = getScalaNodeText(Pair.getKey()); + auto* Value = Pair.getValue(); + if (Key == "clangModuleName") { + entry.moduleName = saver.save(getScalaNodeText(Value)); + entry.isSwift = false; + } else if (Key == "swiftModuleName") { + entry.moduleName = saver.save(getScalaNodeText(Value)); + entry.isSwift = true; + } else if (Key == "arguments") { + entry.arguments = saver.save(getScalaNodeText(Value)); + } else if (Key == "output") { + entry.outputPath = saver.save(getScalaNodeText(Value)); + } else { + // Future proof. + continue; + } + } + if (entry.moduleName.empty()) + return true; + if (entry.outputPath.empty()) + return true; + result.emplace_back(std::move(entry)); + } + return false; +} +static Optional> +parseBatchScanInputFile(ASTContext &ctx, StringRef batchInputPath, + llvm::StringSaver &saver) { + assert(!batchInputPath.empty()); + namespace yaml = llvm::yaml; + std::vector result; + + // Load the input file. + llvm::ErrorOr> FileBufOrErr = + llvm::MemoryBuffer::getFile(batchInputPath); + if (!FileBufOrErr) { + ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_missing, + batchInputPath); + return None; + } + StringRef Buffer = FileBufOrErr->get()->getBuffer(); + + // Use a new source manager instead of the one from ASTContext because we + // don't want the Json file to be persistent. + SourceManager SM; + yaml::Stream Stream(llvm::MemoryBufferRef(Buffer, batchInputPath), + SM.getLLVMSourceMgr()); + for (auto DI = Stream.begin(); DI != Stream.end(); ++ DI) { + assert(DI != Stream.end() && "Failed to read a document"); + yaml::Node *N = DI->getRoot(); + assert(N && "Failed to find a root"); + if (parseBatchInputEntries(ctx, saver, N, result)) { + ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_corrupted, + batchInputPath); + return None; + } + } + return result; +} } /// Find all of the imported Clang modules starting with the given module name. @@ -533,15 +624,17 @@ static bool diagnoseCycle(CompilerInstance &instance, return false; } -bool swift::scanClangDependencies(CompilerInstance &instance) { +static bool scanModuleDependencies(CompilerInstance &instance, + StringRef moduleName, + StringRef arguments, + bool isClang, + StringRef outputPath) { ASTContext &ctx = instance.getASTContext(); - ModuleDecl *mainModule = instance.getMainModule(); auto &FEOpts = instance.getInvocation().getFrontendOptions(); ModuleInterfaceLoaderOptions LoaderOpts(FEOpts); auto ModuleCachePath = getModuleCachePathFromClang(ctx .getClangModuleLoader()->getClangInstance()); - StringRef mainModuleName = mainModule->getNameStr(); llvm::SetVector, std::set> allModules; // Create the module dependency cache. @@ -555,16 +648,23 @@ bool swift::scanClangDependencies(CompilerInstance &instance) { FEOpts.PrebuiltModuleCachePath, FEOpts.SerializeModuleInterfaceDependencyHashes, FEOpts.shouldTrackSystemDependencies()); - // Loading the clang module using Clang importer. - // This action will populate the cache with the main module's dependencies. - auto rootDeps = static_cast(ctx.getClangModuleLoader()) - ->getModuleDependencies(mainModuleName, cache, ASTDelegate); + Optional rootDeps; + if (isClang) { + // Loading the clang module using Clang importer. + // This action will populate the cache with the main module's dependencies. + rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, cache, + ASTDelegate); + } else { + // Loading the swift module's dependencies. + rootDeps = ctx.getSwiftModuleDependencies(moduleName, cache, ASTDelegate); + } if (!rootDeps.hasValue()) { // We cannot find the clang module, abort. return true; } // Add the main module. - allModules.insert({mainModuleName.str(), ModuleDependenciesKind::Clang}); + allModules.insert({moduleName.str(), isClang ? ModuleDependenciesKind::Clang: + ModuleDependenciesKind::Swift}); // Explore the dependencies of every module. for (unsigned currentModuleIdx = 0; @@ -576,13 +676,38 @@ bool swift::scanClangDependencies(CompilerInstance &instance) { allModules.insert(discoveredModules.begin(), discoveredModules.end()); } // Write out the JSON description. - std::string path = FEOpts.InputsAndOutputs.getSingleOutputFilename(); std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); + llvm::raw_fd_ostream out(outputPath, EC, llvm::sys::fs::F_None); writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef()); return false; } +bool swift::scanClangDependencies(CompilerInstance &instance) { + return scanModuleDependencies(instance, + instance.getMainModule()->getNameStr(), + StringRef(), + /*isClang*/true, + instance.getInvocation().getFrontendOptions() + .InputsAndOutputs.getSingleOutputFilename()); +} + +bool swift::batchScanModuleDependencies(CompilerInstance &instance, + llvm::StringRef batchInputFile) { + (void)instance.getMainModule(); + llvm::BumpPtrAllocator alloc; + llvm::StringSaver saver(alloc); + auto results = parseBatchScanInputFile(instance.getASTContext(), + batchInputFile, saver); + if (!results.hasValue()) + return true; + for (auto &entry: *results) { + if (scanModuleDependencies(instance, entry.moduleName, entry.arguments, + !entry.isSwift, entry.outputPath)) + return true; + } + return false; +} + bool swift::scanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); ModuleDecl *mainModule = instance.getMainModule(); diff --git a/lib/FrontendTool/ScanDependencies.h b/lib/FrontendTool/ScanDependencies.h index e3ed171c2badb..d4e0cf1a2c544 100644 --- a/lib/FrontendTool/ScanDependencies.h +++ b/lib/FrontendTool/ScanDependencies.h @@ -13,10 +13,15 @@ #ifndef SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H #define SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H +#include "llvm/ADT/StringRef.h" + namespace swift { class CompilerInstance; +/// Batch scan the dependencies for modules specified in \c batchInputFile. +bool batchScanModuleDependencies(CompilerInstance &instance, llvm::StringRef batchInputFile); + /// Scans the dependencies of the main module of \c instance. bool scanDependencies(CompilerInstance &instance); diff --git a/test/ScanDependencies/batch_module_scan.swift b/test/ScanDependencies/batch_module_scan.swift new file mode 100644 index 0000000000000..08151fc4eb6f6 --- /dev/null +++ b/test/ScanDependencies/batch_module_scan.swift @@ -0,0 +1,39 @@ +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/inputs) +// RUN: %empty-directory(%t/outputs) +// RUN: mkdir -p %t/clang-module-cache + +// RUN: echo "[{" > %/t/inputs/input.json +// RUN: echo "\"swiftModuleName\": \"F\"," >> %/t/inputs/input.json +// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json +// RUN: echo "\"output\": \"%/t/outputs/F.swiftmodule.json\"" >> %/t/inputs/input.json +// RUN: echo "}," >> %/t/inputs/input.json +// RUN: echo "{" >> %/t/inputs/input.json +// RUN: echo "\"clangModuleName\": \"F\"," >> %/t/inputs/input.json +// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json +// RUN: echo "\"output\": \"%/t/outputs/F.pcm.json\"" >> %/t/inputs/input.json +// RUN: echo "}]" >> %/t/inputs/input.json + +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 -batch-scan-input-file %/t/inputs/input.json + +// Check the contents of the JSON output +// RUN: %FileCheck %s -check-prefix=CHECK-PCM < %t/outputs/F.pcm.json +// RUN: %FileCheck %s -check-prefix=CHECK-SWIFT < %t/outputs/F.swiftmodule.json + +// CHECK-PCM: { +// CHECK-PCM-NEXT: "mainModuleName": "F", +// CHECK-PCM-NEXT: "modules": [ +// CHECK-PCM-NEXT: { +// CHECK-PCM-NEXT: "clang": "F" +// CHECK-PCM-NEXT: }, +// CHECK-PCM-NEXT: { +// CHECK-PCM-NEXT: "modulePath": "F.pcm", + +// CHECK-SWIFT: { +// CHECK-SWIFT-NEXT: "mainModuleName": "F", +// CHECK-SWIFT-NEXT: "modules": [ +// CHECK-SWIFT-NEXT: { +// CHECK-SWIFT-NEXT: "swift": "F" +// CHECK-SWIFT-NEXT: }, +// CHECK-SWIFT-NEXT: { +// CHECK-SWIFT-NEXT: "modulePath": "F.swiftmodule",