From 29e655bac03c60a29e1fb95be083a15f557926e4 Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Thu, 20 Aug 2020 11:45:21 -0700 Subject: [PATCH 1/2] DependenciesScanner: implement protocol for batch module dependencies scan This scanning mode allows swift-driver to query module dependencies in a batch and in a more granular way. In short term, it could help solve a problem that clang module dependencies may vary if target triple changes. In a longer term, we could break a holistic dependencies graph into smaller pieces for better caching and reusing. This change doesn't include the implementation of using the specified scanner arguments to set up Clang dependencies scanner. It will come in later commits. --- include/swift/AST/ASTContext.h | 6 + include/swift/AST/DiagnosticsFrontend.def | 8 + include/swift/AST/SearchPathOptions.h | 3 + include/swift/Option/FrontendOptions.td | 4 + lib/AST/ASTContext.cpp | 15 ++ lib/Frontend/CompilerInvocation.cpp | 3 +- lib/FrontendTool/FrontendTool.cpp | 10 +- lib/FrontendTool/ScanDependencies.cpp | 145 ++++++++++++++++-- lib/FrontendTool/ScanDependencies.h | 5 + test/ScanDependencies/batch_module_scan.swift | 39 +++++ 10 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 test/ScanDependencies/batch_module_scan.swift diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h index 9ef666affbf7e..9281ebcf55c2b 100644 --- a/include/swift/AST/ASTContext.h +++ b/include/swift/AST/ASTContext.h @@ -739,6 +739,12 @@ class ASTContext final { ModuleDependenciesCache &cache, InterfaceSubContextDelegate &delegate); + /// Retrieve the module dependencies for the Swift module with the given name. + Optional getSwiftModuleDependencies( + StringRef moduleName, + ModuleDependenciesCache &cache, + InterfaceSubContextDelegate &delegate); + /// Load extensions to the given nominal type from the external /// module loaders. /// diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def index 21931481da7be..36471ecbc9d92 100644 --- a/include/swift/AST/DiagnosticsFrontend.def +++ b/include/swift/AST/DiagnosticsFrontend.def @@ -274,6 +274,14 @@ ERROR(placeholder_dependency_module_map_corrupted,none, "Swift placeholder dependency module map from %0 is malformed", (StringRef)) +ERROR(batch_scan_input_file_missing,none, + "cannot open batch dependencies scan input file from %0", + (StringRef)) + +ERROR(batch_scan_input_file_corrupted,none, + "batch dependencies scan input file from %0 is malformed", + (StringRef)) + REMARK(default_previous_install_name, none, "default previous install name for %0 is %1", (StringRef, StringRef)) diff --git a/include/swift/AST/SearchPathOptions.h b/include/swift/AST/SearchPathOptions.h index 08757ffd796de..52eeace6a1460 100644 --- a/include/swift/AST/SearchPathOptions.h +++ b/include/swift/AST/SearchPathOptions.h @@ -93,6 +93,9 @@ class SearchPathOptions { /// A map of placeholder Swift module dependency information. std::string PlaceholderDependencyModuleMap; + + /// A file containing modules we should perform batch scanning. + std::string BatchScanInputFilePath; private: static StringRef pathStringFromFrameworkSearchPath(const FrameworkSearchPath &next) { diff --git a/include/swift/Option/FrontendOptions.td b/include/swift/Option/FrontendOptions.td index 5bd693d2ffe7d..8c00c8a94ca16 100644 --- a/include/swift/Option/FrontendOptions.td +++ b/include/swift/Option/FrontendOptions.td @@ -228,6 +228,10 @@ def explict_swift_module_map def placeholder_dependency_module_map : Separate<["-"], "placeholder-dependency-module-map-file">, MetaVarName<"">, HelpText<"Specify a JSON file containing information of external Swift module dependencies">; + +def batch_scan_input_file + : Separate<["-"], "batch-scan-input-file">, MetaVarName<"">, + HelpText<"Specify a JSON file containing modules to perform batch dependencies scanning">; } diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp index e75c5185cc13b..f3ee9197874a9 100644 --- a/lib/AST/ASTContext.cpp +++ b/lib/AST/ASTContext.cpp @@ -1516,6 +1516,21 @@ Optional ASTContext::getModuleDependencies( return None; } +Optional +ASTContext::getSwiftModuleDependencies(StringRef moduleName, + ModuleDependenciesCache &cache, + InterfaceSubContextDelegate &delegate) { + for (auto &loader : getImpl().ModuleLoaders) { + if (loader.get() == getImpl().TheClangModuleLoader) + continue; + + if (auto dependencies = loader->getModuleDependencies(moduleName, cache, + delegate)) + return dependencies; + } + return None; +} + void ASTContext::loadExtensions(NominalTypeDecl *nominal, unsigned previousGeneration) { PrettyStackTraceDecl stackTrace("loading extensions for", nominal); diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp index f9c3828c19c67..db5894b1d4a20 100644 --- a/lib/Frontend/CompilerInvocation.cpp +++ b/lib/Frontend/CompilerInvocation.cpp @@ -926,7 +926,8 @@ static bool ParseSearchPathArgs(SearchPathOptions &Opts, } if (const Arg *A = Args.getLastArg(OPT_placeholder_dependency_module_map)) Opts.PlaceholderDependencyModuleMap = A->getValue(); - + if (const Arg *A = Args.getLastArg(OPT_batch_scan_input_file)) + Opts.BatchScanInputFilePath = A->getValue(); // Opts.RuntimeIncludePath is set by calls to // setRuntimeIncludePath() or setMainExecutablePath(). // Opts.RuntimeImportPath is set by calls to diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index 9bdfe3e1da24b..063e89e2860d6 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -1833,8 +1833,14 @@ static bool performCompile(CompilerInstance &Instance, return finishPipeline(Context.hadError()); } - if (Action == FrontendOptions::ActionType::ScanDependencies) - return finishPipeline(scanDependencies(Instance)); + if (Action == FrontendOptions::ActionType::ScanDependencies) { + auto batchScanInput = Instance.getASTContext().SearchPathOpts.BatchScanInputFilePath; + if (batchScanInput.empty()) + return finishPipeline(scanDependencies(Instance)); + else + return finishPipeline(batchScanModuleDependencies(Instance, + batchScanInput)); + } if (Action == FrontendOptions::ActionType::ScanClangDependencies) return finishPipeline(scanClangDependencies(Instance)); diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index bf2fccc25de93..4a616a69a53dc 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -32,12 +32,101 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/YAMLParser.h" #include using namespace swift; +using namespace llvm::yaml; namespace { +struct BatchScanInput { + StringRef moduleName; + StringRef arguments; + StringRef outputPath; +}; + +static std::string getScalaNodeText(Node *N) { + SmallString<32> Buffer; + return cast(N)->getValue(Buffer).str(); +} + +/// Parse an entry like this, where the "platforms" key-value pair is optional: +/// { +/// "module": "Foo.pcm", +/// "arguments": "-target 10.15", +/// "output": "../Foo.json" +/// }, +static bool parseBatchInputEntries(ASTContext &Ctx, llvm::StringSaver &saver, + Node *Node, std::vector &result) { + auto *SN = cast(Node); + if (!SN) + return true; + for (auto It = SN->begin(); It != SN->end(); ++It) { + auto *MN = cast(&*It); + BatchScanInput entry; + Optional> Platforms; + for (auto &Pair: *MN) { + auto Key = getScalaNodeText(Pair.getKey()); + auto* Value = Pair.getValue(); + if (Key == "module") { + entry.moduleName = saver.save(getScalaNodeText(Value)); + } else if (Key == "arguments") { + entry.arguments = saver.save(getScalaNodeText(Value)); + } else if (Key == "output") { + entry.outputPath = saver.save(getScalaNodeText(Value)); + } else { + // Future proof. + continue; + } + } + if (entry.moduleName.empty()) + return true; + if (entry.outputPath.empty()) + return true; + auto ext = llvm::sys::path::extension(entry.moduleName); + if (ext != ".swiftmodule" && ext != ".pcm") + return true; + result.emplace_back(std::move(entry)); + } + return false; +} +static Optional> +parseBatchScanInputFile(ASTContext &ctx, StringRef batchInputPath, + llvm::StringSaver &saver) { + assert(!batchInputPath.empty()); + namespace yaml = llvm::yaml; + std::vector result; + + // Load the input file. + llvm::ErrorOr> FileBufOrErr = + llvm::MemoryBuffer::getFile(batchInputPath); + if (!FileBufOrErr) { + ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_missing, + batchInputPath); + return None; + } + StringRef Buffer = FileBufOrErr->get()->getBuffer(); + + // Use a new source manager instead of the one from ASTContext because we + // don't want the Json file to be persistent. + SourceManager SM; + yaml::Stream Stream(llvm::MemoryBufferRef(Buffer, batchInputPath), + SM.getLLVMSourceMgr()); + for (auto DI = Stream.begin(); DI != Stream.end(); ++ DI) { + assert(DI != Stream.end() && "Failed to read a document"); + yaml::Node *N = DI->getRoot(); + assert(N && "Failed to find a root"); + if (parseBatchInputEntries(ctx, saver, N, result)) { + ctx.Diags.diagnose(SourceLoc(), diag::batch_scan_input_file_corrupted, + batchInputPath); + return None; + } + } + return result; +} } /// Find all of the imported Clang modules starting with the given module name. @@ -533,15 +622,17 @@ static bool diagnoseCycle(CompilerInstance &instance, return false; } -bool swift::scanClangDependencies(CompilerInstance &instance) { +static bool scanModuleDependencies(CompilerInstance &instance, + StringRef moduleName, + StringRef arguments, + bool isClang, + StringRef outputPath) { ASTContext &ctx = instance.getASTContext(); - ModuleDecl *mainModule = instance.getMainModule(); auto &FEOpts = instance.getInvocation().getFrontendOptions(); ModuleInterfaceLoaderOptions LoaderOpts(FEOpts); auto ModuleCachePath = getModuleCachePathFromClang(ctx .getClangModuleLoader()->getClangInstance()); - StringRef mainModuleName = mainModule->getNameStr(); llvm::SetVector, std::set> allModules; // Create the module dependency cache. @@ -555,16 +646,23 @@ bool swift::scanClangDependencies(CompilerInstance &instance) { FEOpts.PrebuiltModuleCachePath, FEOpts.SerializeModuleInterfaceDependencyHashes, FEOpts.shouldTrackSystemDependencies()); - // Loading the clang module using Clang importer. - // This action will populate the cache with the main module's dependencies. - auto rootDeps = static_cast(ctx.getClangModuleLoader()) - ->getModuleDependencies(mainModuleName, cache, ASTDelegate); + Optional rootDeps; + if (isClang) { + // Loading the clang module using Clang importer. + // This action will populate the cache with the main module's dependencies. + rootDeps = ctx.getModuleDependencies(moduleName, /*IsClang*/true, cache, + ASTDelegate); + } else { + // Loading the swift module's dependencies. + rootDeps = ctx.getSwiftModuleDependencies(moduleName, cache, ASTDelegate); + } if (!rootDeps.hasValue()) { // We cannot find the clang module, abort. return true; } // Add the main module. - allModules.insert({mainModuleName.str(), ModuleDependenciesKind::Clang}); + allModules.insert({moduleName.str(), isClang ? ModuleDependenciesKind::Clang: + ModuleDependenciesKind::Swift}); // Explore the dependencies of every module. for (unsigned currentModuleIdx = 0; @@ -576,13 +674,40 @@ bool swift::scanClangDependencies(CompilerInstance &instance) { allModules.insert(discoveredModules.begin(), discoveredModules.end()); } // Write out the JSON description. - std::string path = FEOpts.InputsAndOutputs.getSingleOutputFilename(); std::error_code EC; - llvm::raw_fd_ostream out(path, EC, llvm::sys::fs::F_None); + llvm::raw_fd_ostream out(outputPath, EC, llvm::sys::fs::F_None); writeJSON(out, instance, cache, ASTDelegate, allModules.getArrayRef()); return false; } +bool swift::scanClangDependencies(CompilerInstance &instance) { + return scanModuleDependencies(instance, + instance.getMainModule()->getNameStr(), + StringRef(), + /*isClang*/true, + instance.getInvocation().getFrontendOptions() + .InputsAndOutputs.getSingleOutputFilename()); +} + +bool swift::batchScanModuleDependencies(CompilerInstance &instance, + llvm::StringRef batchInputFile) { + (void)instance.getMainModule(); + llvm::BumpPtrAllocator alloc; + llvm::StringSaver saver(alloc); + auto results = parseBatchScanInputFile(instance.getASTContext(), + batchInputFile, saver); + if (!results.hasValue()) + return true; + for (auto &entry: *results) { + auto moduleName = llvm::sys::path::stem(entry.moduleName); + auto isClang = llvm::sys::path::extension(entry.moduleName) == ".pcm"; + if (scanModuleDependencies(instance, moduleName, entry.arguments, isClang, + entry.outputPath)) + return true; + } + return false; +} + bool swift::scanDependencies(CompilerInstance &instance) { ASTContext &Context = instance.getASTContext(); ModuleDecl *mainModule = instance.getMainModule(); diff --git a/lib/FrontendTool/ScanDependencies.h b/lib/FrontendTool/ScanDependencies.h index e3ed171c2badb..d4e0cf1a2c544 100644 --- a/lib/FrontendTool/ScanDependencies.h +++ b/lib/FrontendTool/ScanDependencies.h @@ -13,10 +13,15 @@ #ifndef SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H #define SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H +#include "llvm/ADT/StringRef.h" + namespace swift { class CompilerInstance; +/// Batch scan the dependencies for modules specified in \c batchInputFile. +bool batchScanModuleDependencies(CompilerInstance &instance, llvm::StringRef batchInputFile); + /// Scans the dependencies of the main module of \c instance. bool scanDependencies(CompilerInstance &instance); diff --git a/test/ScanDependencies/batch_module_scan.swift b/test/ScanDependencies/batch_module_scan.swift new file mode 100644 index 0000000000000..a6fb01313fedb --- /dev/null +++ b/test/ScanDependencies/batch_module_scan.swift @@ -0,0 +1,39 @@ +// RUN: %empty-directory(%t) +// RUN: %empty-directory(%t/inputs) +// RUN: %empty-directory(%t/outputs) +// RUN: mkdir -p %t/clang-module-cache + +// RUN: echo "[{" > %/t/inputs/input.json +// RUN: echo "\"module\": \"F.swiftmodule\"," >> %/t/inputs/input.json +// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json +// RUN: echo "\"output\": \"%/t/outputs/F.swiftmodule.json\"" >> %/t/inputs/input.json +// RUN: echo "}," >> %/t/inputs/input.json +// RUN: echo "{" >> %/t/inputs/input.json +// RUN: echo "\"module\": \"F.pcm\"," >> %/t/inputs/input.json +// RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json +// RUN: echo "\"output\": \"%/t/outputs/F.pcm.json\"" >> %/t/inputs/input.json +// RUN: echo "}]" >> %/t/inputs/input.json + +// RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 -batch-scan-input-file %/t/inputs/input.json + +// Check the contents of the JSON output +// RUN: %FileCheck %s -check-prefix=CHECK-PCM < %t/outputs/F.pcm.json +// RUN: %FileCheck %s -check-prefix=CHECK-SWIFT < %t/outputs/F.swiftmodule.json + +// CHECK-PCM: { +// CHECK-PCM-NEXT: "mainModuleName": "F", +// CHECK-PCM-NEXT: "modules": [ +// CHECK-PCM-NEXT: { +// CHECK-PCM-NEXT: "clang": "F" +// CHECK-PCM-NEXT: }, +// CHECK-PCM-NEXT: { +// CHECK-PCM-NEXT: "modulePath": "F.pcm", + +// CHECK-SWIFT: { +// CHECK-SWIFT-NEXT: "mainModuleName": "F", +// CHECK-SWIFT-NEXT: "modules": [ +// CHECK-SWIFT-NEXT: { +// CHECK-SWIFT-NEXT: "swift": "F" +// CHECK-SWIFT-NEXT: }, +// CHECK-SWIFT-NEXT: { +// CHECK-SWIFT-NEXT: "modulePath": "F.swiftmodule", From 6a6b1f832d2c9cc9f3a89f2d38729845edeb08ac Mon Sep 17 00:00:00 2001 From: Xi Ge Date: Thu, 20 Aug 2020 14:40:54 -0700 Subject: [PATCH 2/2] DependenciesScanner: differentiate Swift and Clang module more explicitly in batch scan input --- lib/FrontendTool/ScanDependencies.cpp | 18 +++++++++--------- test/ScanDependencies/batch_module_scan.swift | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/FrontendTool/ScanDependencies.cpp b/lib/FrontendTool/ScanDependencies.cpp index 4a616a69a53dc..524a5de2213e6 100644 --- a/lib/FrontendTool/ScanDependencies.cpp +++ b/lib/FrontendTool/ScanDependencies.cpp @@ -45,6 +45,7 @@ struct BatchScanInput { StringRef moduleName; StringRef arguments; StringRef outputPath; + bool isSwift; }; static std::string getScalaNodeText(Node *N) { @@ -54,7 +55,7 @@ static std::string getScalaNodeText(Node *N) { /// Parse an entry like this, where the "platforms" key-value pair is optional: /// { -/// "module": "Foo.pcm", +/// "swiftModuleName": "Foo", /// "arguments": "-target 10.15", /// "output": "../Foo.json" /// }, @@ -70,8 +71,12 @@ static bool parseBatchInputEntries(ASTContext &Ctx, llvm::StringSaver &saver, for (auto &Pair: *MN) { auto Key = getScalaNodeText(Pair.getKey()); auto* Value = Pair.getValue(); - if (Key == "module") { + if (Key == "clangModuleName") { entry.moduleName = saver.save(getScalaNodeText(Value)); + entry.isSwift = false; + } else if (Key == "swiftModuleName") { + entry.moduleName = saver.save(getScalaNodeText(Value)); + entry.isSwift = true; } else if (Key == "arguments") { entry.arguments = saver.save(getScalaNodeText(Value)); } else if (Key == "output") { @@ -85,9 +90,6 @@ static bool parseBatchInputEntries(ASTContext &Ctx, llvm::StringSaver &saver, return true; if (entry.outputPath.empty()) return true; - auto ext = llvm::sys::path::extension(entry.moduleName); - if (ext != ".swiftmodule" && ext != ".pcm") - return true; result.emplace_back(std::move(entry)); } return false; @@ -699,10 +701,8 @@ bool swift::batchScanModuleDependencies(CompilerInstance &instance, if (!results.hasValue()) return true; for (auto &entry: *results) { - auto moduleName = llvm::sys::path::stem(entry.moduleName); - auto isClang = llvm::sys::path::extension(entry.moduleName) == ".pcm"; - if (scanModuleDependencies(instance, moduleName, entry.arguments, isClang, - entry.outputPath)) + if (scanModuleDependencies(instance, entry.moduleName, entry.arguments, + !entry.isSwift, entry.outputPath)) return true; } return false; diff --git a/test/ScanDependencies/batch_module_scan.swift b/test/ScanDependencies/batch_module_scan.swift index a6fb01313fedb..08151fc4eb6f6 100644 --- a/test/ScanDependencies/batch_module_scan.swift +++ b/test/ScanDependencies/batch_module_scan.swift @@ -4,12 +4,12 @@ // RUN: mkdir -p %t/clang-module-cache // RUN: echo "[{" > %/t/inputs/input.json -// RUN: echo "\"module\": \"F.swiftmodule\"," >> %/t/inputs/input.json +// RUN: echo "\"swiftModuleName\": \"F\"," >> %/t/inputs/input.json // RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json // RUN: echo "\"output\": \"%/t/outputs/F.swiftmodule.json\"" >> %/t/inputs/input.json // RUN: echo "}," >> %/t/inputs/input.json // RUN: echo "{" >> %/t/inputs/input.json -// RUN: echo "\"module\": \"F.pcm\"," >> %/t/inputs/input.json +// RUN: echo "\"clangModuleName\": \"F\"," >> %/t/inputs/input.json // RUN: echo "\"arguments\": \"-target x86_64-apple-macosx10.9\"," >> %/t/inputs/input.json // RUN: echo "\"output\": \"%/t/outputs/F.pcm.json\"" >> %/t/inputs/input.json // RUN: echo "}]" >> %/t/inputs/input.json