From 9055a401a30ff496761d2c580b5ef8bf112b4bfd Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Wed, 20 Nov 2024 13:02:22 -0800 Subject: [PATCH] [Dependency Scanner] Refactor the global scanning service to no longer maintain scanner cache state Instead, each scan's 'ModuleDependenciesCache' will hold all of the data corresponding to discovered module dependencies. The initial design presumed the possibility of sharing a global scanning cache amongs different scanner invocations, possibly even different concurrent scanner invocations. This change also deprecates two libSwiftScan entry-points: 'swiftscan_scanner_cache_load' and 'swiftscan_scanner_cache_serialize'. They never ended up getting used, and since this code has been largely stale, we are confident they have not otherwise had users, and they do not fit with this design. A follow-up change will re-introduce moduele dependency cache serialization on a per-query basis and bring the binary format up-to-date. --- .../swift-c/DependencyScan/DependencyScan.h | 15 +- include/swift/AST/ModuleDependencies.h | 111 +---- .../DependencyScan/DependencyScanningTool.h | 5 - .../SerializedModuleDependencyCacheFormat.h | 8 +- lib/AST/ModuleDependencies.cpp | 186 ++------ lib/DependencyScan/DependencyScanningTool.cpp | 25 -- .../ModuleDependencyCacheSerialization.cpp | 420 ++++++++---------- lib/DependencyScan/ScanDependencies.cpp | 22 +- lib/Tooling/libSwiftScan/libSwiftScan.cpp | 12 - lib/Tooling/libSwiftScan/libSwiftScan.exports | 2 - test/Frontend/output_determinism_check.swift | 3 +- .../module_deps_cache_reuse.swift | 2 + ...module_deps_different_paths_no_reuse.swift | 2 + 13 files changed, 276 insertions(+), 537 deletions(-) diff --git a/include/swift-c/DependencyScan/DependencyScan.h b/include/swift-c/DependencyScan/DependencyScan.h index 4e2132cdbc021..1c8ab8e771233 100644 --- a/include/swift-c/DependencyScan/DependencyScan.h +++ b/include/swift-c/DependencyScan/DependencyScan.h @@ -24,8 +24,8 @@ /// The version constants for the SwiftDependencyScan C API. /// SWIFTSCAN_VERSION_MINOR should increase when there are API additions. /// SWIFTSCAN_VERSION_MAJOR is intended for "major" source/ABI breaking changes. -#define SWIFTSCAN_VERSION_MAJOR 0 -#define SWIFTSCAN_VERSION_MINOR 10 +#define SWIFTSCAN_VERSION_MAJOR 1 +#define SWIFTSCAN_VERSION_MINOR 0 SWIFTSCAN_BEGIN_DECLS @@ -481,17 +481,6 @@ swiftscan_source_location_get_column_number(swiftscan_source_location_t source_l // to allow clients to perform incremental dependency scans by having the // scanner's state be serializable and re-usable. -/// For the specified \c scanner instance, serialize its state to the specified file-system \c path . -SWIFTSCAN_PUBLIC void -swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner, - const char * path); - -/// For the specified \c scanner instance, load in scanner state from a file at -/// the specified file-system \c path . -SWIFTSCAN_PUBLIC bool -swiftscan_scanner_cache_load(swiftscan_scanner_t scanner, - const char * path); - /// For the specified \c scanner instance, reset its internal state, ensuring subsequent /// scanning queries are done "from-scratch". SWIFTSCAN_PUBLIC void diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 61684ff4e4cde..cc46f833b44ec 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -971,10 +971,6 @@ using ModuleNameToDependencyMap = llvm::StringMap; using ModuleDependenciesKindMap = std::unordered_map; -using ModuleDependenciesKindRefMap = - std::unordered_map, - ModuleDependencyKindHash>; // MARK: SwiftDependencyTracker /// Track swift dependency @@ -1005,23 +1001,8 @@ class SwiftDependencyTracker { // MARK: SwiftDependencyScanningService /// A carrier of state shared among possibly multiple invocations of the -/// dependency scanner. Acts as a global cache of discovered module dependencies -/// and filesystem state. It is not to be queried directly, but is rather meant -/// to be wrapped in an instance of `ModuleDependenciesCache`, responsible for -/// recording new dependencies and answering cache queries in a given scan. +/// dependency scanner. class SwiftDependencyScanningService { - /// Global cache contents specific to a specific scanner invocation context - struct ContextSpecificGlobalCacheState { - /// All cached module dependencies, in the order in which they were - /// encountered. - std::vector AllModules; - - /// Dependencies for modules that have already been computed. - /// This maps a dependency kind to a map of a module's name to a Dependency - /// object - ModuleDependenciesKindMap ModuleDependenciesMap; - }; - /// The CASOption created the Scanning Service if used. std::optional CASOpts; @@ -1046,27 +1027,9 @@ class SwiftDependencyScanningService { clang::tooling::dependencies::DependencyScanningFilesystemSharedCache> SharedFilesystemCache; - /// A map from a String representing the target triple of a scanner invocation - /// to the corresponding cached dependencies discovered so far when using this - /// triple. - llvm::StringMap> - ContextSpecificCacheMap; - - /// The context hashes used by scanners using this cache, in the order in - /// which they were used - std::vector AllContextHashes; - /// Shared state mutual-exclusivity lock mutable llvm::sys::SmartMutex ScanningServiceGlobalLock; - /// Retrieve the dependencies map that corresponds to the given dependency - /// kind. - ModuleNameToDependencyMap &getDependenciesMap(ModuleDependencyKind kind, - StringRef scanContextHash); - const ModuleNameToDependencyMap & - getDependenciesMap(ModuleDependencyKind kind, - StringRef scanContextHash) const; - public: SwiftDependencyScanningService(); SwiftDependencyScanningService(const SwiftDependencyScanningService &) = @@ -1136,54 +1099,6 @@ class SwiftDependencyScanningService { friend class ModuleDependenciesCache; friend class ModuleDependencyScanner; friend class ModuleDependencyScanningWorker; - friend class ModuleDependenciesCacheDeserializer; - friend class ModuleDependenciesCacheSerializer; - - /// Configure the current state of the cache to respond to queries - /// for the specified scanning context hash. - void configureForContextHash(StringRef scanningContextHash); - - /// Return context hashes of all scanner invocations that have used - /// this cache instance. - const std::vector &getAllContextHashes() const { - return AllContextHashes; - } - - /// Whether we have cached dependency information for the given module. - bool hasDependency(StringRef moduleName, - std::optional kind, - StringRef scanContextHash) const; - - /// Return a pointer to the cache state of the specified context hash. - ContextSpecificGlobalCacheState * - getCacheForScanningContextHash(StringRef scanContextHash) const; - - /// Look for module dependencies for a module with the given name - /// - /// \returns the cached result, or \c None if there is no cached entry. - std::optional - findDependency(StringRef moduleName, std::optional kind, - StringRef scanContextHash) const; - - /// Record dependencies for the given module. - const ModuleDependencyInfo * - recordDependency(StringRef moduleName, ModuleDependencyInfo dependencies, - StringRef scanContextHash); - - /// Update stored dependencies for the given module. - const ModuleDependencyInfo * - updateDependency(ModuleDependencyID moduleID, - ModuleDependencyInfo dependencies, - StringRef scanContextHash); - - /// Reference the list of all module dependency infos for a given scanning - /// context - const std::vector & - getAllModules(StringRef scanningContextHash) const { - auto contextSpecificCache = - getCacheForScanningContextHash(scanningContextHash); - return contextSpecificCache->AllModules; - } }; // MARK: ModuleDependenciesCache @@ -1195,11 +1110,10 @@ class SwiftDependencyScanningService { class ModuleDependenciesCache { private: SwiftDependencyScanningService &globalScanningService; - /// References to data in the `globalScanningService` for module dependencies - ModuleDependenciesKindRefMap ModuleDependenciesMap; + /// Discovered dependencies + ModuleDependenciesKindMap ModuleDependenciesMap; /// Set containing all of the Clang modules that have already been seen. - llvm::DenseSet - alreadySeenClangModules; + llvm::DenseSet alreadySeenClangModules; /// Name of the module under scan std::string mainScanModuleName; /// The context hash of the current scanning invocation @@ -1223,6 +1137,11 @@ class ModuleDependenciesCache { ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete; public: + /// Retrieve the dependencies map that corresponds to the given dependency + /// kind. + ModuleNameToDependencyMap &getDependenciesMap(ModuleDependencyKind kind); + const ModuleNameToDependencyMap &getDependenciesMap(ModuleDependencyKind kind) const; + /// Whether we have cached dependency information for the given module. bool hasDependency(const ModuleDependencyID &moduleID) const; /// Whether we have cached dependency information for the given module. @@ -1283,13 +1202,7 @@ class ModuleDependenciesCache { /// \returns the cached result, or \c None if there is no cached entry. std::optional findDependency(StringRef moduleName, - std::optional kind) const; - - /// Look for module dependencies for a module with the given name - /// - /// \returns the cached result, or \c None if there is no cached entry. - std::optional - findDependency(StringRef moduleName) const; + std::optional kind = std::nullopt) const; /// Look for Swift module dependencies for a module with the given name /// @@ -1340,6 +1253,10 @@ class ModuleDependenciesCache { const ArrayRef dependencyIDs); StringRef getMainModuleName() const { return mainScanModuleName; } + +private: + friend class ModuleDependenciesCacheDeserializer; + friend class ModuleDependenciesCacheSerializer; }; } // namespace swift diff --git a/include/swift/DependencyScan/DependencyScanningTool.h b/include/swift/DependencyScan/DependencyScanningTool.h index 71f3160f063cf..528d00517f80a 100644 --- a/include/swift/DependencyScan/DependencyScanningTool.h +++ b/include/swift/DependencyScan/DependencyScanningTool.h @@ -108,11 +108,6 @@ class DependencyScanningTool { const llvm::StringSet<> &PlaceholderModules, StringRef WorkingDirectory); - /// Writes the current `SharedCache` instance to a specified FileSystem path. - void serializeCache(llvm::StringRef path); - /// Loads an instance of a `SwiftDependencyScanningService` to serve as the `SharedCache` - /// from a specified FileSystem path. - bool loadCache(llvm::StringRef path); /// Discard the tool's current `SharedCache` and start anew. void resetCache(); /// Query diagnostics consumed so far. diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index 546d38bc3de1f..aac398fd68821 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -209,24 +209,24 @@ using ClangModuleDetailsLayout = /// Tries to read the dependency graph from the given buffer. /// Returns \c true if there was an error. bool readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, - SwiftDependencyScanningService &cache); + ModuleDependenciesCache &cache); /// Tries to read the dependency graph from the given path name. /// Returns true if there was an error. bool readInterModuleDependenciesCache(llvm::StringRef path, - SwiftDependencyScanningService &cache); + ModuleDependenciesCache &cache); /// Tries to write the dependency graph to the given path name. /// Returns true if there was an error. bool writeInterModuleDependenciesCache(DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, llvm::StringRef path, - const SwiftDependencyScanningService &cache); + const ModuleDependenciesCache &cache); /// Tries to write out the given dependency cache with the given /// bitstream writer. void writeInterModuleDependenciesCache(llvm::BitstreamWriter &Out, - const SwiftDependencyScanningService &cache); + const ModuleDependenciesCache &cache); } // end namespace module_dependency_cache_serialization } // end namespace dependencies diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp index a878e2c108d18..be5c103ac649b 100644 --- a/lib/AST/ModuleDependencies.cpp +++ b/lib/AST/ModuleDependencies.cpp @@ -719,147 +719,67 @@ bool SwiftDependencyScanningService::setupCachingDependencyScanningService( return false; } -SwiftDependencyScanningService::ContextSpecificGlobalCacheState * -SwiftDependencyScanningService::getCacheForScanningContextHash(StringRef scanningContextHash) const { - llvm::sys::SmartScopedLock Lock(ScanningServiceGlobalLock); - auto contextSpecificCache = ContextSpecificCacheMap.find(scanningContextHash); - assert(contextSpecificCache != ContextSpecificCacheMap.end() && - "Global Module Dependencies Cache not configured with context-specific " - "state."); - return contextSpecificCache->getValue().get(); +ModuleDependenciesCache::ModuleDependenciesCache( + SwiftDependencyScanningService &globalScanningService, + std::string mainScanModuleName, std::string moduleOutputPath, + std::string scannerContextHash) + : globalScanningService(globalScanningService), + mainScanModuleName(mainScanModuleName), + scannerContextHash(scannerContextHash), + moduleOutputPath(moduleOutputPath) { + for (auto kind = ModuleDependencyKind::FirstKind; + kind != ModuleDependencyKind::LastKind; ++kind) + ModuleDependenciesMap.insert({kind, ModuleNameToDependencyMap()}); } const ModuleNameToDependencyMap & -SwiftDependencyScanningService::getDependenciesMap( - ModuleDependencyKind kind, StringRef scanContextHash) const { - auto contextSpecificCache = getCacheForScanningContextHash(scanContextHash); - auto it = contextSpecificCache->ModuleDependenciesMap.find(kind); - assert(it != contextSpecificCache->ModuleDependenciesMap.end() && +ModuleDependenciesCache::getDependenciesMap( + ModuleDependencyKind kind) const { + auto it = ModuleDependenciesMap.find(kind); + assert(it != ModuleDependenciesMap.end() && "invalid dependency kind"); return it->second; } ModuleNameToDependencyMap & -SwiftDependencyScanningService::getDependenciesMap( - ModuleDependencyKind kind, StringRef scanContextHash) { - auto contextSpecificCache = getCacheForScanningContextHash(scanContextHash); - auto it = contextSpecificCache->ModuleDependenciesMap.find(kind); - assert(it != contextSpecificCache->ModuleDependenciesMap.end() && +ModuleDependenciesCache::getDependenciesMap( + ModuleDependencyKind kind) { + auto it = ModuleDependenciesMap.find(kind); + assert(it != ModuleDependenciesMap.end() && "invalid dependency kind"); return it->second; } -void SwiftDependencyScanningService::configureForContextHash(StringRef scanningContextHash) { - llvm::sys::SmartScopedLock Lock(ScanningServiceGlobalLock); - auto knownContext = ContextSpecificCacheMap.find(scanningContextHash); - if (knownContext == ContextSpecificCacheMap.end()) { - // First time scanning with this context, initialize context-specific state. - std::unique_ptr contextSpecificCache = - std::make_unique(); - for (auto kind = ModuleDependencyKind::FirstKind; - kind != ModuleDependencyKind::LastKind; ++kind) { - contextSpecificCache->ModuleDependenciesMap.insert({kind, ModuleNameToDependencyMap()}); - } - ContextSpecificCacheMap.insert({scanningContextHash.str(), std::move(contextSpecificCache)}); - AllContextHashes.push_back(scanningContextHash.str()); - } +std::optional +ModuleDependenciesCache::findDependency( + const ModuleDependencyID moduleID) const { + return findDependency(moduleID.ModuleName, moduleID.Kind); } std::optional -SwiftDependencyScanningService::findDependency( - StringRef moduleName, std::optional kind, - StringRef scanningContextHash) const { +ModuleDependenciesCache::findDependency( + StringRef moduleName, std::optional kind) const { if (!kind) { for (auto kind = ModuleDependencyKind::FirstKind; kind != ModuleDependencyKind::LastKind; ++kind) { - auto dep = findDependency(moduleName, kind, scanningContextHash); + auto dep = findDependency(moduleName, kind); if (dep.has_value()) return dep.value(); } return std::nullopt; } - + assert(kind.has_value() && "Expected dependencies kind for lookup."); - const auto &map = getDependenciesMap(kind.value(), scanningContextHash); + std::optional optionalDep = std::nullopt; + const auto &map = getDependenciesMap(kind.value()); auto known = map.find(moduleName); if (known != map.end()) - return &(known->second); - - return std::nullopt; -} - -bool SwiftDependencyScanningService::hasDependency( - StringRef moduleName, std::optional kind, - StringRef scanContextHash) const { - return findDependency(moduleName, kind, scanContextHash).has_value(); -} - -const ModuleDependencyInfo *SwiftDependencyScanningService::recordDependency( - StringRef moduleName, ModuleDependencyInfo dependencies, - StringRef scanContextHash) { - auto kind = dependencies.getKind(); - auto &map = getDependenciesMap(kind, scanContextHash); - map.insert({moduleName, dependencies}); - return &(map[moduleName]); -} - -const ModuleDependencyInfo *SwiftDependencyScanningService::updateDependency( - ModuleDependencyID moduleID, ModuleDependencyInfo dependencies, - StringRef scanningContextHash) { - auto &map = getDependenciesMap(moduleID.Kind, scanningContextHash); - auto known = map.find(moduleID.ModuleName); - assert(known != map.end() && "Not yet added to map"); - known->second = std::move(dependencies); - return &(known->second); -} - -llvm::StringMap & -ModuleDependenciesCache::getDependencyReferencesMap( - ModuleDependencyKind kind) { - auto it = ModuleDependenciesMap.find(kind); - assert(it != ModuleDependenciesMap.end() && "invalid dependency kind"); - return it->second; -} - -const llvm::StringMap & -ModuleDependenciesCache::getDependencyReferencesMap( - ModuleDependencyKind kind) const { - auto it = ModuleDependenciesMap.find(kind); - assert(it != ModuleDependenciesMap.end() && "invalid dependency kind"); - return it->second; -} - -ModuleDependenciesCache::ModuleDependenciesCache( - SwiftDependencyScanningService &globalScanningService, - std::string mainScanModuleName, std::string moduleOutputPath, - std::string scannerContextHash) - : globalScanningService(globalScanningService), - mainScanModuleName(mainScanModuleName), - scannerContextHash(scannerContextHash), - moduleOutputPath(moduleOutputPath) { - globalScanningService.configureForContextHash(scannerContextHash); - for (auto kind = ModuleDependencyKind::FirstKind; - kind != ModuleDependencyKind::LastKind; ++kind) { - ModuleDependenciesMap.insert( - {kind, llvm::StringMap()}); - } -} - -std::optional -ModuleDependenciesCache::findDependency( - const ModuleDependencyID moduleID) const { - return findDependency(moduleID.ModuleName, moduleID.Kind); -} - -std::optional -ModuleDependenciesCache::findDependency( - StringRef moduleName, std::optional kind) const { - auto optionalDep = globalScanningService.findDependency(moduleName, kind, - scannerContextHash); + optionalDep = &(known->second); + // During a scan, only produce the cached source module info for the current // module under scan. - if (optionalDep) { - auto dep = *optionalDep; + if (optionalDep.has_value()) { + auto dep = optionalDep.value(); if (dep->getAsSwiftSourceModule() && moduleName != mainScanModuleName && moduleName != "DummyMainModuleForResolvingCrossImportOverlays") { @@ -870,16 +790,6 @@ ModuleDependenciesCache::findDependency( return optionalDep; } -std::optional -ModuleDependenciesCache::findDependency(StringRef moduleName) const { - for (auto kind = ModuleDependencyKind::FirstKind; - kind != ModuleDependencyKind::LastKind; ++kind) { - if (auto found = findDependency(moduleName, kind)) - return found; - } - return std::nullopt; -} - std::optional ModuleDependenciesCache::findSwiftDependency(StringRef moduleName) const { if (auto found = findDependency(moduleName, ModuleDependencyKind::SwiftInterface)) @@ -911,10 +821,9 @@ bool ModuleDependenciesCache::hasDependency( bool ModuleDependenciesCache::hasDependency(StringRef moduleName) const { for (auto kind = ModuleDependencyKind::FirstKind; - kind != ModuleDependencyKind::LastKind; ++kind) { + kind != ModuleDependencyKind::LastKind; ++kind) if (findDependency(moduleName, kind).has_value()) return true; - } return false; } @@ -923,19 +832,15 @@ bool ModuleDependenciesCache::hasSwiftDependency(StringRef moduleName) const { } void ModuleDependenciesCache::recordDependency( - StringRef moduleName, ModuleDependencyInfo dependencies) { - auto dependenciesKind = dependencies.getKind(); - const ModuleDependencyInfo *recordedDependencies = - globalScanningService.recordDependency(moduleName, dependencies, - scannerContextHash); - auto &map = getDependencyReferencesMap(dependenciesKind); - assert(map.count(moduleName) == 0 && "Already added to map"); - map.insert({moduleName, recordedDependencies}); + StringRef moduleName, ModuleDependencyInfo dependency) { + auto dependenciesKind = dependency.getKind(); + auto &map = getDependenciesMap(dependenciesKind); + map.insert({moduleName, dependency}); } void ModuleDependenciesCache::recordDependencies( - ModuleDependencyVector moduleDependencies) { - for (const auto &dep : moduleDependencies) { + ModuleDependencyVector dependencies) { + for (const auto &dep : dependencies) { if (!hasDependency(dep.first)) recordDependency(dep.first.ModuleName, dep.second); if (dep.second.getKind() == ModuleDependencyKind::Clang) { @@ -948,14 +853,9 @@ void ModuleDependenciesCache::recordDependencies( void ModuleDependenciesCache::updateDependency( ModuleDependencyID moduleID, ModuleDependencyInfo dependencyInfo) { - const ModuleDependencyInfo *updatedDependencies = - globalScanningService.updateDependency(moduleID, dependencyInfo, - scannerContextHash); - auto &map = getDependencyReferencesMap(moduleID.Kind); - auto known = map.find(moduleID.ModuleName); - if (known != map.end()) - map.erase(known); - map.insert({moduleID.ModuleName, updatedDependencies}); + auto &map = getDependenciesMap(moduleID.Kind); + assert(map.find(moduleID.ModuleName) != map.end() && "Not yet added to map"); + map.insert_or_assign(moduleID.ModuleName, std::move(dependencyInfo)); } void diff --git a/lib/DependencyScan/DependencyScanningTool.cpp b/lib/DependencyScan/DependencyScanningTool.cpp index 358c78dbc63ef..dcdd3deb56f01 100644 --- a/lib/DependencyScan/DependencyScanningTool.cpp +++ b/lib/DependencyScan/DependencyScanningTool.cpp @@ -356,31 +356,6 @@ DependencyScanningTool::getDependencies( return BatchScanResults; } -void DependencyScanningTool::serializeCache(llvm::StringRef path) { - llvm::sys::SmartScopedLock Lock(DependencyScanningToolStateLock); - SourceManager SM; - DiagnosticEngine Diags(SM); - Diags.addConsumer(CDC); - llvm::vfs::OnDiskOutputBackend Backend; - module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Diags, Backend, path, *ScanningService); -} - -bool DependencyScanningTool::loadCache(llvm::StringRef path) { - llvm::sys::SmartScopedLock Lock(DependencyScanningToolStateLock); - SourceManager SM; - DiagnosticEngine Diags(SM); - Diags.addConsumer(CDC); - ScanningService = std::make_unique(); - bool readFailed = - module_dependency_cache_serialization::readInterModuleDependenciesCache( - path, *ScanningService); - if (readFailed) { - Diags.diagnose(SourceLoc(), diag::warn_scanner_deserialize_failed, path); - } - return readFailed; -} - void DependencyScanningTool::resetCache() { llvm::sys::SmartScopedLock Lock(DependencyScanningToolStateLock); ScanningService.reset(new SwiftDependencyScanningService()); diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 5060d52cc65e9..5bd4b0173752e 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -38,7 +38,7 @@ class ModuleDependenciesCacheDeserializer { bool readSignature(); bool enterGraphBlock(); bool readMetadata(); - bool readGraph(SwiftDependencyScanningService &cache); + bool readGraph(ModuleDependenciesCache &cache); std::optional getIdentifier(unsigned n); std::optional> getStringArray(unsigned n); @@ -47,7 +47,7 @@ class ModuleDependenciesCacheDeserializer { public: ModuleDependenciesCacheDeserializer(llvm::MemoryBufferRef Data) : Cursor(Data) {} - bool readInterModuleDependenciesCache(SwiftDependencyScanningService &cache); + bool readInterModuleDependenciesCache(ModuleDependenciesCache &cache); }; } // end namespace @@ -150,7 +150,7 @@ bool ModuleDependenciesCacheDeserializer::readMetadata() { /// all of the file's identifiers and arrays of identifiers, followed by /// consuming individual module info records and registering them into the /// cache. -bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningService &cache) { +bool ModuleDependenciesCacheDeserializer::readGraph(ModuleDependenciesCache &cache) { using namespace graph_block; bool hasCurrentModule = false; @@ -160,15 +160,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi std::vector currentOptionalModuleImports; std::vector currentModuleDependencyIDs; - auto getContextHash = [&]() { - assert(currentContextHashID && - "Expected context hash ID for a MODULE_DETAILS_NODE record"); - auto contextHash = getIdentifier(currentContextHashID); - if (!contextHash.has_value()) - llvm::report_fatal_error("Unexpected MODULE_DETAILS_NODE record"); - return contextHash.value(); - }; - while (!Cursor.AtEndOfStream()) { auto entry = cantFail(Cursor.advance(), "Advance bitstream cursor"); @@ -249,7 +240,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi if (!hasCurrentModule) llvm::report_fatal_error( "Unexpected SWIFT_TEXTUAL_MODULE_DETAILS_NODE record"); - cache.configureForContextHash(getContextHash()); unsigned outputPathFileID, interfaceFileID, compiledModuleCandidatesArrayID, buildCommandLineArrayID, extraPCMArgsArrayID, contextHashID, isFramework, isStatic, bridgingHeaderFileID, @@ -384,8 +374,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi if (!bridgingHeaderIncludeTree->empty()) moduleDep.addBridgingHeaderIncludeTree(*bridgingHeaderIncludeTree); - cache.recordDependency(currentModuleName, std::move(moduleDep), - getContextHash()); + cache.recordDependency(currentModuleName, std::move(moduleDep)); hasCurrentModule = false; break; } @@ -495,8 +484,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi if (!bridgingHeaderIncludeTree->empty()) moduleDep.addBridgingHeaderIncludeTree(*bridgingHeaderIncludeTree); - cache.recordDependency(currentModuleName, std::move(moduleDep), - getContextHash()); + cache.recordDependency(currentModuleName, std::move(moduleDep)); hasCurrentModule = false; break; } @@ -505,7 +493,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi if (!hasCurrentModule) llvm::report_fatal_error( "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record"); - cache.configureForContextHash(getContextHash()); unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID, overlayDependencyIDArrayID, headerImportID, headerModuleDependenciesArrayID, @@ -568,8 +555,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi llvm::report_fatal_error("Bad overlay dependencies: no qualified dependencies"); moduleDep.setSwiftOverlayDependencies(*overlayModuleDependencyIDs); - cache.recordDependency(currentModuleName, std::move(moduleDep), - getContextHash()); + cache.recordDependency(currentModuleName, std::move(moduleDep)); hasCurrentModule = false; break; } @@ -578,7 +564,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi if (!hasCurrentModule) llvm::report_fatal_error( "Unexpected SWIFT_PLACEHOLDER_MODULE_DETAILS_NODE record"); - cache.configureForContextHash(getContextHash()); unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID; SwiftPlaceholderModuleDetailsLayout::readRecord( Scratch, compiledModulePathID, moduleDocPathID, @@ -605,8 +590,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi for (const auto &moduleName : currentOptionalModuleImports) moduleDep.addOptionalModuleImport(moduleName.importIdentifier); - cache.recordDependency(currentModuleName, std::move(moduleDep), - getContextHash()); + cache.recordDependency(currentModuleName, std::move(moduleDep)); hasCurrentModule = false; break; } @@ -614,7 +598,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi case CLANG_MODULE_DETAILS_NODE: { if (!hasCurrentModule) llvm::report_fatal_error("Unexpected CLANG_MODULE_DETAILS_NODE record"); - cache.configureForContextHash(getContextHash()); unsigned pcmOutputPathID, mappedPCMPathID, moduleMapPathID, contextHashID, commandLineArrayID, fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID, clangIncludeTreeRootID, moduleCacheKeyID, isSystem; @@ -668,8 +651,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi for (const auto &moduleName : currentOptionalModuleImports) moduleDep.addOptionalModuleImport(moduleName.importIdentifier); - cache.recordDependency(currentModuleName, std::move(moduleDep), - getContextHash()); + cache.recordDependency(currentModuleName, std::move(moduleDep)); hasCurrentModule = false; break; } @@ -684,7 +666,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi } bool ModuleDependenciesCacheDeserializer::readInterModuleDependenciesCache( - SwiftDependencyScanningService &cache) { + ModuleDependenciesCache &cache) { using namespace graph_block; if (readSignature()) @@ -772,14 +754,14 @@ ModuleDependenciesCacheDeserializer::getModuleDependencyIDArray(unsigned n) { bool swift::dependencies::module_dependency_cache_serialization:: readInterModuleDependenciesCache(llvm::MemoryBuffer &buffer, - SwiftDependencyScanningService &cache) { + ModuleDependenciesCache &cache) { ModuleDependenciesCacheDeserializer deserializer(buffer.getMemBufferRef()); return deserializer.readInterModuleDependenciesCache(cache); } bool swift::dependencies::module_dependency_cache_serialization:: readInterModuleDependenciesCache(StringRef path, - SwiftDependencyScanningService &cache) { + ModuleDependenciesCache &cache) { PrettyStackTraceStringAction stackTrace( "loading inter-module dependency graph", path); auto buffer = llvm::MemoryBuffer::getFile(path); @@ -881,7 +863,7 @@ class ModuleDependenciesCacheSerializer { AbbrCodes[Layout::Code] = Layout::emitAbbrev(Out); } - void collectStringsAndArrays(const SwiftDependencyScanningService &cache); + void collectStringsAndArrays(const ModuleDependenciesCache &cache); void emitBlockID(unsigned ID, StringRef name, SmallVectorImpl &nameBuffer); @@ -897,7 +879,6 @@ class ModuleDependenciesCacheSerializer { void writeArraysOfIdentifiers(); void writeModuleInfo(ModuleDependencyID moduleID, - std::optional contextHash, const ModuleDependencyInfo &dependencyInfo); public: @@ -905,7 +886,7 @@ class ModuleDependenciesCacheSerializer { public: void - writeInterModuleDependenciesCache(const SwiftDependencyScanningService &cache); + writeInterModuleDependenciesCache(const ModuleDependenciesCache &cache); }; } // end namespace @@ -987,10 +968,11 @@ void ModuleDependenciesCacheSerializer::writeArraysOfIdentifiers() { } void ModuleDependenciesCacheSerializer::writeModuleInfo( - ModuleDependencyID moduleID, std::optional contextHash, + ModuleDependencyID moduleID, const ModuleDependencyInfo &dependencyInfo) { using namespace graph_block; - auto contextHashStrID = contextHash.has_value() ? getIdentifier(contextHash.value()) : 0; + // TODO: Eliminate per-module context hash + auto contextHashStrID = 0; ModuleInfoLayout::emitRecord( Out, ScratchRecord, AbbrCodes[ModuleInfoLayout::Code], @@ -1001,7 +983,6 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo( switch (dependencyInfo.getKind()) { case swift::ModuleDependencyKind::SwiftInterface: { - assert(contextHash.has_value() && "Expected context hash for serializing MODULE_NODE"); auto swiftTextDeps = dependencyInfo.getAsSwiftInterfaceModule(); assert(swiftTextDeps); unsigned outputModulePathFileId = @@ -1037,8 +1018,6 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo( break; } case swift::ModuleDependencyKind::SwiftSource: { - assert(!contextHash.has_value() && - "Did not expect context hash for serializing MODULE_NODE"); auto swiftSourceDeps = dependencyInfo.getAsSwiftSourceModule(); assert(swiftSourceDeps); unsigned bridgingHeaderFileId = @@ -1066,7 +1045,6 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo( break; } case swift::ModuleDependencyKind::SwiftBinary: { - assert(contextHash.has_value() && "Expected context hash for serializing MODULE_NODE"); auto swiftBinDeps = dependencyInfo.getAsSwiftBinaryModule(); assert(swiftBinDeps); SwiftBinaryModuleDetailsLayout::emitRecord( @@ -1085,7 +1063,6 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo( break; } case swift::ModuleDependencyKind::SwiftPlaceholder: { - assert(contextHash.has_value() && "Expected context hash for serializing MODULE_NODE"); auto swiftPHDeps = dependencyInfo.getAsPlaceholderDependencyModule(); assert(swiftPHDeps); SwiftPlaceholderModuleDetailsLayout::emitRecord( @@ -1097,7 +1074,6 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo( break; } case swift::ModuleDependencyKind::Clang: { - assert(contextHash.has_value() && "Expected context hash for serializing MODULE_NODE"); auto clangDeps = dependencyInfo.getAsClangModule(); assert(clangDeps); ClangModuleDetailsLayout::emitRecord( @@ -1153,8 +1129,7 @@ void ModuleDependenciesCacheSerializer::addDependencyIDArray(ModuleDependencyID std::vector encodedDependencyIDs; for (const auto &moduleID : vec) encodedDependencyIDs.push_back(createEncodedModuleKindAndName(moduleID)); - addStringArray(moduleID, ModuleIdentifierArrayKind::QualifiedModuleDependencyIDs, - encodedDependencyIDs); + addStringArray(moduleID, arrayKind, encodedDependencyIDs); return; } @@ -1199,186 +1174,186 @@ unsigned ModuleDependenciesCacheSerializer::getArrayID(ModuleDependencyID module } void ModuleDependenciesCacheSerializer::collectStringsAndArrays( - const SwiftDependencyScanningService &cache) { - for (auto &contextHash : cache.getAllContextHashes()) { - addIdentifier(contextHash); - for (auto &moduleID : cache.getAllModules(contextHash)) { - auto optionalDependencyInfo = - cache.findDependency(moduleID.ModuleName, moduleID.Kind, contextHash); - assert(optionalDependencyInfo.has_value() && "Expected dependency info."); - auto dependencyInfo = optionalDependencyInfo.value(); - // Add the module's name - addIdentifier(moduleID.ModuleName); - - // Map import infos to their respective module identifiers - auto importInfoArrayToIdentifier = - [](const auto &importInfo) -> std::string { - return importInfo.importIdentifier; - }; - - // Add the module's dependencies - std::vector importIdentifiers; - llvm::transform(dependencyInfo->getModuleImports(), - std::back_inserter(importIdentifiers), - importInfoArrayToIdentifier); - std::vector optionalImportIdentifiers; - llvm::transform(dependencyInfo->getOptionalModuleImports(), - std::back_inserter(optionalImportIdentifiers), - importInfoArrayToIdentifier); - - addStringArray(moduleID, ModuleIdentifierArrayKind::DependencyImports, - importIdentifiers); - addStringArray(moduleID, ModuleIdentifierArrayKind::OptionalDependencyImports, - optionalImportIdentifiers); - - ModuleDependencyIDSetVector allDependencies; - if (dependencyInfo->isSwiftModule()) { - auto swiftImportedDepsRef = dependencyInfo->getImportedSwiftDependencies(); - auto headerClangDepsRef = dependencyInfo->getHeaderClangDependencies(); - auto overlayDependenciesRef = dependencyInfo->getSwiftOverlayDependencies(); - allDependencies.insert(swiftImportedDepsRef.begin(), - swiftImportedDepsRef.end()); - allDependencies.insert(headerClangDepsRef.begin(), - headerClangDepsRef.end()); - allDependencies.insert(overlayDependenciesRef.begin(), - overlayDependenciesRef.end()); - } - - if (dependencyInfo->isSwiftSourceModule()) { - auto crossImportOverlayDepsRef = dependencyInfo->getCrossImportOverlayDependencies(); - allDependencies.insert(crossImportOverlayDepsRef.begin(), - crossImportOverlayDepsRef.end()); - } + const ModuleDependenciesCache &cache) { + addIdentifier(cache.scannerContextHash); + // TODO: Serialize *all* modules + for (auto &moduleID : cache.getAllDependencies({cache.mainScanModuleName, + ModuleDependencyKind::SwiftSource})) { + auto optionalDependencyInfo = + cache.findDependency(moduleID.ModuleName, moduleID.Kind); + assert(optionalDependencyInfo.has_value() && "Expected dependency info."); + auto dependencyInfo = optionalDependencyInfo.value(); + // Add the module's name + addIdentifier(moduleID.ModuleName); + + // Map import infos to their respective module identifiers + auto importInfoArrayToIdentifier = + [](const auto &importInfo) -> std::string { + return importInfo.importIdentifier; + }; + + // Add the module's dependencies + std::vector importIdentifiers; + llvm::transform(dependencyInfo->getModuleImports(), + std::back_inserter(importIdentifiers), + importInfoArrayToIdentifier); + std::vector optionalImportIdentifiers; + llvm::transform(dependencyInfo->getOptionalModuleImports(), + std::back_inserter(optionalImportIdentifiers), + importInfoArrayToIdentifier); + + addStringArray(moduleID, ModuleIdentifierArrayKind::DependencyImports, + importIdentifiers); + addStringArray(moduleID, ModuleIdentifierArrayKind::OptionalDependencyImports, + optionalImportIdentifiers); + + ModuleDependencyIDSetVector allDependencies; + if (dependencyInfo->isSwiftModule()) { + auto swiftImportedDepsRef = dependencyInfo->getImportedSwiftDependencies(); + auto headerClangDepsRef = dependencyInfo->getHeaderClangDependencies(); + auto overlayDependenciesRef = dependencyInfo->getSwiftOverlayDependencies(); + allDependencies.insert(swiftImportedDepsRef.begin(), + swiftImportedDepsRef.end()); + allDependencies.insert(headerClangDepsRef.begin(), + headerClangDepsRef.end()); + allDependencies.insert(overlayDependenciesRef.begin(), + overlayDependenciesRef.end()); + } - auto clangImportedDepsRef = dependencyInfo->getImportedClangDependencies(); - allDependencies.insert(clangImportedDepsRef.begin(), - clangImportedDepsRef.end()); + if (dependencyInfo->isSwiftSourceModule()) { + auto crossImportOverlayDepsRef = dependencyInfo->getCrossImportOverlayDependencies(); + allDependencies.insert(crossImportOverlayDepsRef.begin(), + crossImportOverlayDepsRef.end()); + } + auto clangImportedDepsRef = dependencyInfo->getImportedClangDependencies(); + allDependencies.insert(clangImportedDepsRef.begin(), + clangImportedDepsRef.end()); + + addDependencyIDArray( + moduleID, ModuleIdentifierArrayKind::QualifiedModuleDependencyIDs, + allDependencies.getArrayRef()); + + std::vector clangHeaderDependencyNames; + for (const auto &headerDepID : + dependencyInfo->getHeaderClangDependencies()) + clangHeaderDependencyNames.push_back(headerDepID.ModuleName); + + // Add the dependency-kind-specific data + switch (dependencyInfo->getKind()) { + case swift::ModuleDependencyKind::SwiftInterface: { + auto swiftTextDeps = dependencyInfo->getAsSwiftInterfaceModule(); + assert(swiftTextDeps); + addIdentifier(swiftTextDeps->moduleOutputPath); + addIdentifier(swiftTextDeps->swiftInterfaceFile); + addStringArray(moduleID, + ModuleIdentifierArrayKind::CompiledModuleCandidates, + swiftTextDeps->compiledModuleCandidates); + addStringArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine, + swiftTextDeps->textualModuleDetails.buildCommandLine); + addStringArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, + swiftTextDeps->textualModuleDetails.extraPCMArgs); + addIdentifier(swiftTextDeps->contextHash); + if (swiftTextDeps->textualModuleDetails.bridgingHeaderFile.has_value()) + addIdentifier(swiftTextDeps->textualModuleDetails.bridgingHeaderFile + .value()); + addStringArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, + std::vector()); + addStringArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, + swiftTextDeps->textualModuleDetails.bridgingSourceFiles); + addStringArray( + moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, + clangHeaderDependencyNames); addDependencyIDArray( - moduleID, ModuleIdentifierArrayKind::QualifiedModuleDependencyIDs, - allDependencies.getArrayRef()); - - std::vector clangHeaderDependencyNames; - for (const auto &headerDepID : - dependencyInfo->getHeaderClangDependencies()) - clangHeaderDependencyNames.push_back(headerDepID.ModuleName); - - // Add the dependency-kind-specific data - switch (dependencyInfo->getKind()) { - case swift::ModuleDependencyKind::SwiftInterface: { - auto swiftTextDeps = dependencyInfo->getAsSwiftInterfaceModule(); - assert(swiftTextDeps); - addIdentifier(swiftTextDeps->moduleOutputPath); - addIdentifier(swiftTextDeps->swiftInterfaceFile); - addStringArray(moduleID, - ModuleIdentifierArrayKind::CompiledModuleCandidates, - swiftTextDeps->compiledModuleCandidates); - addStringArray(moduleID, ModuleIdentifierArrayKind::BuildCommandLine, - swiftTextDeps->textualModuleDetails.buildCommandLine); - addStringArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, - swiftTextDeps->textualModuleDetails.extraPCMArgs); - addIdentifier(swiftTextDeps->contextHash); - if (swiftTextDeps->textualModuleDetails.bridgingHeaderFile.has_value()) - addIdentifier(swiftTextDeps->textualModuleDetails.bridgingHeaderFile - .value()); - addStringArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, - std::vector()); - addStringArray(moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, - swiftTextDeps->textualModuleDetails.bridgingSourceFiles); - addStringArray( - moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, - clangHeaderDependencyNames); - addDependencyIDArray( - moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, - swiftTextDeps->swiftOverlayDependencies); - addIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID); - addIdentifier(swiftTextDeps->textualModuleDetails - .CASBridgingHeaderIncludeTreeRootID); - addIdentifier(swiftTextDeps->moduleCacheKey); - break; - } - case swift::ModuleDependencyKind::SwiftBinary: { - auto swiftBinDeps = dependencyInfo->getAsSwiftBinaryModule(); - assert(swiftBinDeps); - addIdentifier(swiftBinDeps->compiledModulePath); - addIdentifier(swiftBinDeps->moduleDocPath); - addIdentifier(swiftBinDeps->sourceInfoPath); - addIdentifier(swiftBinDeps->moduleCacheKey); - addIdentifier(swiftBinDeps->headerImport); - addStringArray(moduleID, ModuleIdentifierArrayKind::HeaderInputModuleDependencies, - clangHeaderDependencyNames); - addStringArray(moduleID, ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles, - swiftBinDeps->headerSourceFiles); - addDependencyIDArray( - moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, - swiftBinDeps->swiftOverlayDependencies); - break; - } - case swift::ModuleDependencyKind::SwiftPlaceholder: { - auto swiftPHDeps = dependencyInfo->getAsPlaceholderDependencyModule(); - assert(swiftPHDeps); - addIdentifier(swiftPHDeps->compiledModulePath); - addIdentifier(swiftPHDeps->moduleDocPath); - addIdentifier(swiftPHDeps->sourceInfoPath); - break; - } - case swift::ModuleDependencyKind::SwiftSource: { - auto swiftSourceDeps = dependencyInfo->getAsSwiftSourceModule(); - assert(swiftSourceDeps); - addStringArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, - swiftSourceDeps->textualModuleDetails.extraPCMArgs); - if (swiftSourceDeps->textualModuleDetails.bridgingHeaderFile - .has_value()) - addIdentifier( - swiftSourceDeps->textualModuleDetails.bridgingHeaderFile.value()); - addStringArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, - swiftSourceDeps->sourceFiles); - addStringArray( - moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, - swiftSourceDeps->textualModuleDetails.bridgingSourceFiles); - addStringArray( - moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, - clangHeaderDependencyNames); - addDependencyIDArray( - moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, - swiftSourceDeps->swiftOverlayDependencies); - addStringArray( - moduleID, ModuleIdentifierArrayKind::BuildCommandLine, - swiftSourceDeps->textualModuleDetails.buildCommandLine); - addStringArray( - moduleID, ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine, - swiftSourceDeps->bridgingHeaderBuildCommandLine); + moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, + swiftTextDeps->swiftOverlayDependencies); + addIdentifier(swiftTextDeps->textualModuleDetails.CASFileSystemRootID); + addIdentifier(swiftTextDeps->textualModuleDetails + .CASBridgingHeaderIncludeTreeRootID); + addIdentifier(swiftTextDeps->moduleCacheKey); + break; + } + case swift::ModuleDependencyKind::SwiftBinary: { + auto swiftBinDeps = dependencyInfo->getAsSwiftBinaryModule(); + assert(swiftBinDeps); + addIdentifier(swiftBinDeps->compiledModulePath); + addIdentifier(swiftBinDeps->moduleDocPath); + addIdentifier(swiftBinDeps->sourceInfoPath); + addIdentifier(swiftBinDeps->moduleCacheKey); + addIdentifier(swiftBinDeps->headerImport); + addStringArray(moduleID, ModuleIdentifierArrayKind::HeaderInputModuleDependencies, + clangHeaderDependencyNames); + addStringArray(moduleID, ModuleIdentifierArrayKind::HeaderInputDependencySourceFiles, + swiftBinDeps->headerSourceFiles); + addDependencyIDArray( + moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, + swiftBinDeps->swiftOverlayDependencies); + break; + } + case swift::ModuleDependencyKind::SwiftPlaceholder: { + auto swiftPHDeps = dependencyInfo->getAsPlaceholderDependencyModule(); + assert(swiftPHDeps); + addIdentifier(swiftPHDeps->compiledModulePath); + addIdentifier(swiftPHDeps->moduleDocPath); + addIdentifier(swiftPHDeps->sourceInfoPath); + break; + } + case swift::ModuleDependencyKind::SwiftSource: { + auto swiftSourceDeps = dependencyInfo->getAsSwiftSourceModule(); + assert(swiftSourceDeps); + addStringArray(moduleID, ModuleIdentifierArrayKind::ExtraPCMArgs, + swiftSourceDeps->textualModuleDetails.extraPCMArgs); + if (swiftSourceDeps->textualModuleDetails.bridgingHeaderFile + .has_value()) addIdentifier( - swiftSourceDeps->textualModuleDetails.CASFileSystemRootID); - break; - } - case swift::ModuleDependencyKind::Clang: { - auto clangDeps = dependencyInfo->getAsClangModule(); - assert(clangDeps); - addIdentifier(clangDeps->pcmOutputPath); - addIdentifier(clangDeps->mappedPCMPath); - addIdentifier(clangDeps->moduleMapFile); - addIdentifier(clangDeps->contextHash); - addStringArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine, - clangDeps->buildCommandLine); - addStringArray(moduleID, ModuleIdentifierArrayKind::FileDependencies, - clangDeps->fileDependencies); - addStringArray(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs, - clangDeps->capturedPCMArgs); - addIdentifier(clangDeps->CASFileSystemRootID); - addIdentifier(clangDeps->CASClangIncludeTreeRootID); - addIdentifier(clangDeps->moduleCacheKey); - break; - } - default: - llvm_unreachable("Unhandled dependency kind."); - } + swiftSourceDeps->textualModuleDetails.bridgingHeaderFile.value()); + addStringArray(moduleID, ModuleIdentifierArrayKind::SourceFiles, + swiftSourceDeps->sourceFiles); + addStringArray( + moduleID, ModuleIdentifierArrayKind::BridgingSourceFiles, + swiftSourceDeps->textualModuleDetails.bridgingSourceFiles); + addStringArray( + moduleID, ModuleIdentifierArrayKind::BridgingModuleDependencies, + clangHeaderDependencyNames); + addDependencyIDArray( + moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependencyIDs, + swiftSourceDeps->swiftOverlayDependencies); + addStringArray( + moduleID, ModuleIdentifierArrayKind::BuildCommandLine, + swiftSourceDeps->textualModuleDetails.buildCommandLine); + addStringArray( + moduleID, ModuleIdentifierArrayKind::BridgingHeaderBuildCommandLine, + swiftSourceDeps->bridgingHeaderBuildCommandLine); + addIdentifier( + swiftSourceDeps->textualModuleDetails.CASFileSystemRootID); + break; + } + case swift::ModuleDependencyKind::Clang: { + auto clangDeps = dependencyInfo->getAsClangModule(); + assert(clangDeps); + addIdentifier(clangDeps->pcmOutputPath); + addIdentifier(clangDeps->mappedPCMPath); + addIdentifier(clangDeps->moduleMapFile); + addIdentifier(clangDeps->contextHash); + addStringArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine, + clangDeps->buildCommandLine); + addStringArray(moduleID, ModuleIdentifierArrayKind::FileDependencies, + clangDeps->fileDependencies); + addStringArray(moduleID, ModuleIdentifierArrayKind::CapturedPCMArgs, + clangDeps->capturedPCMArgs); + addIdentifier(clangDeps->CASFileSystemRootID); + addIdentifier(clangDeps->CASClangIncludeTreeRootID); + addIdentifier(clangDeps->moduleCacheKey); + break; + } + default: + llvm_unreachable("Unhandled dependency kind."); } } } void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache( - const SwiftDependencyScanningService &cache) { + const ModuleDependenciesCache &cache) { // Write the header writeSignature(); writeBlockInfoBlock(); @@ -1413,23 +1388,20 @@ void ModuleDependenciesCacheSerializer::writeInterModuleDependenciesCache( writeArraysOfIdentifiers(); // Write the core graph - for (auto &contextHash : cache.getAllContextHashes()) { - for (auto &moduleID : cache.getAllModules(contextHash)) { - auto dependencyInfo = cache.findDependency(moduleID.ModuleName, - moduleID.Kind, - contextHash); - assert(dependencyInfo.has_value() && "Expected dependency info."); - writeModuleInfo(moduleID, contextHash, **dependencyInfo); - } + // TODO: Serialize *all* modules + for (auto &moduleID : cache.getAllDependencies({cache.mainScanModuleName, + ModuleDependencyKind::SwiftSource})) { + auto dependencyInfo = cache.findDependency(moduleID.ModuleName, moduleID.Kind); + assert(dependencyInfo.has_value() && "Expected dependency info."); + writeModuleInfo(moduleID, **dependencyInfo); } - return; } void swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache( llvm::BitstreamWriter &Out, - const SwiftDependencyScanningService &cache) { + const ModuleDependenciesCache &cache) { ModuleDependenciesCacheSerializer serializer{Out}; serializer.writeInterModuleDependenciesCache(cache); } @@ -1437,7 +1409,7 @@ void swift::dependencies::module_dependency_cache_serialization:: bool swift::dependencies::module_dependency_cache_serialization:: writeInterModuleDependenciesCache( DiagnosticEngine &diags, llvm::vfs::OutputBackend &backend, - StringRef path, const SwiftDependencyScanningService &cache) { + StringRef path, const ModuleDependenciesCache &cache) { PrettyStackTraceStringAction stackTrace( "saving inter-module dependency graph", path); return withOutputPath(diags, backend, path, [&](llvm::raw_ostream &out) { diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index e3019644e058f..385d8f737e888 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -1263,24 +1263,24 @@ forEachBatchEntry(CompilerInstance &invocationInstance, } // namespace static void serializeDependencyCache(CompilerInstance &instance, - const SwiftDependencyScanningService &service) { + const ModuleDependenciesCache &cache) { const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); ASTContext &Context = instance.getASTContext(); auto savePath = opts.SerializedDependencyScannerCachePath; module_dependency_cache_serialization::writeInterModuleDependenciesCache( - Context.Diags, instance.getOutputBackend(), savePath, service); + Context.Diags, instance.getOutputBackend(), savePath, cache); if (opts.EmitDependencyScannerCacheRemarks) { Context.Diags.diagnose(SourceLoc(), diag::remark_save_cache, savePath); } } static void deserializeDependencyCache(CompilerInstance &instance, - SwiftDependencyScanningService &service) { + ModuleDependenciesCache &cache) { const FrontendOptions &opts = instance.getInvocation().getFrontendOptions(); ASTContext &Context = instance.getASTContext(); auto loadPath = opts.SerializedDependencyScannerCachePath; if (module_dependency_cache_serialization::readInterModuleDependenciesCache( - loadPath, service)) { + loadPath, cache)) { Context.Diags.diagnose(SourceLoc(), diag::warn_scanner_deserialize_failed, loadPath); } else if (opts.EmitDependencyScannerCacheRemarks) { @@ -1295,16 +1295,18 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { // `-scan-dependencies` invocations use a single new instance // of a module cache SwiftDependencyScanningService *service = Context.Allocate(); + ModuleDependenciesCache cache( + *service, instance.getMainModule()->getNameStr().str(), + instance.getInvocation().getFrontendOptions().ExplicitModulesOutputPath, + instance.getInvocation().getModuleScanningHash()); + if (opts.ReuseDependencyScannerCache) - deserializeDependencyCache(instance, *service); + deserializeDependencyCache(instance, cache); if (service->setupCachingDependencyScanningService(instance)) return true; - ModuleDependenciesCache cache( - *service, instance.getMainModule()->getNameStr().str(), - instance.getInvocation().getFrontendOptions().ExplicitModulesOutputPath, - instance.getInvocation().getModuleScanningHash()); + // Execute scan llvm::ErrorOr dependenciesOrErr = @@ -1313,7 +1315,7 @@ bool swift::dependencies::scanDependencies(CompilerInstance &instance) { // Serialize the dependency cache if -serialize-dependency-scan-cache // is specified if (opts.SerializeDependencyScannerCache) - serializeDependencyCache(instance, *service); + serializeDependencyCache(instance, cache); if (dependenciesOrErr.getError()) return true; diff --git a/lib/Tooling/libSwiftScan/libSwiftScan.cpp b/lib/Tooling/libSwiftScan/libSwiftScan.cpp index 5207bafab177c..67ef444540bb8 100644 --- a/lib/Tooling/libSwiftScan/libSwiftScan.cpp +++ b/lib/Tooling/libSwiftScan/libSwiftScan.cpp @@ -145,18 +145,6 @@ void swiftscan_dependency_set_dispose(swiftscan_dependency_set_t *set) { //=== Scanner Cache Operations --------------------------------------------===// -void swiftscan_scanner_cache_serialize(swiftscan_scanner_t scanner, - const char * path) { - DependencyScanningTool *ScanningTool = unwrap(scanner); - ScanningTool->serializeCache(path); -} - -bool swiftscan_scanner_cache_load(swiftscan_scanner_t scanner, - const char * path) { - DependencyScanningTool *ScanningTool = unwrap(scanner); - return ScanningTool->loadCache(path); -} - void swiftscan_scanner_cache_reset(swiftscan_scanner_t scanner) { DependencyScanningTool *ScanningTool = unwrap(scanner); ScanningTool->resetCache(); diff --git a/lib/Tooling/libSwiftScan/libSwiftScan.exports b/lib/Tooling/libSwiftScan/libSwiftScan.exports index fe59e642a75e1..086feb99324ce 100644 --- a/lib/Tooling/libSwiftScan/libSwiftScan.exports +++ b/lib/Tooling/libSwiftScan/libSwiftScan.exports @@ -77,8 +77,6 @@ swiftscan_compiler_supported_arguments_query swiftscan_compiler_supported_features_query swiftscan_compiler_target_info_query swiftscan_compiler_target_info_query_v2 -swiftscan_scanner_cache_serialize -swiftscan_scanner_cache_load swiftscan_scanner_cache_reset swiftscan_scanner_diagnostics_query swiftscan_scanner_diagnostics_reset diff --git a/test/Frontend/output_determinism_check.swift b/test/Frontend/output_determinism_check.swift index b3fc66f7f6ac6..d5f945e39f6b9 100644 --- a/test/Frontend/output_determinism_check.swift +++ b/test/Frontend/output_determinism_check.swift @@ -12,7 +12,7 @@ /// FAIL: %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files /// Explicit module build. Check building swiftmodule from interface file. -// RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-deterministic-check -load-dependency-scan-cache -dependency-scan-cache-path %t/deps-cache -serialize-dependency-scan-cache 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT --check-prefix=DEPSCAN_CACHE_OUTPUT +// RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT /// TODO: Implicit module build use a different compiler instance so it doesn't support checking yet. // RUN: %target-swift-frontend -typecheck -emit-module-interface-path %t/test.swiftinterface %s -O -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=INTERFACE_OUTPUT /// Hit cache and not emit the second time. @@ -32,7 +32,6 @@ // DEPS_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.d' // OBJECT_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.o' // OBJECT_MISMATCH: error: output file '{{.*}}{{/|\\}}test.o' is missing from second compilation for deterministic check -// DEPSCAN_CACHE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}deps-cache' // DEPSCAN_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.json' // INTERFACE_OUTPUT: remark: produced matching output file '{{.*}}{{/|\\}}test.swiftinterface' // MODULE_MISMATCH: error: output file '{{.*}}{{/|\\}}test.swiftmodule' is missing from second compilation for deterministic check diff --git a/test/ScanDependencies/module_deps_cache_reuse.swift b/test/ScanDependencies/module_deps_cache_reuse.swift index 62db6b6fa533d..df2f9029adf84 100644 --- a/test/ScanDependencies/module_deps_cache_reuse.swift +++ b/test/ScanDependencies/module_deps_cache_reuse.swift @@ -1,5 +1,7 @@ // RUN: %empty-directory(%t) // RUN: mkdir -p %t/clang-module-cache +// Temporarily disabled while the cache serialization code is being brought back from being stale/disabled for a long time +// XFAIL: * // Run the scanner once, emitting the serialized scanner cache // RUN: %target-swift-frontend -scan-dependencies -module-load-mode prefer-interface -Rdependency-scan-cache -serialize-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/clang-module-cache %s -o %t/deps_initial.json -I %S/Inputs/CHeaders -I %S/Inputs/Swift -import-objc-header %S/Inputs/CHeaders/Bridging.h -swift-version 4 -enable-cross-import-overlays 2>&1 | %FileCheck %s -check-prefix CHECK-REMARK-SAVE diff --git a/test/ScanDependencies/module_deps_different_paths_no_reuse.swift b/test/ScanDependencies/module_deps_different_paths_no_reuse.swift index f0d76411b46d0..f15c17bda2723 100644 --- a/test/ScanDependencies/module_deps_different_paths_no_reuse.swift +++ b/test/ScanDependencies/module_deps_different_paths_no_reuse.swift @@ -1,5 +1,7 @@ // RUN: %empty-directory(%t) // RUN: mkdir -p %t/clang-module-cache +// Temporarily disabled while the cache serialization code is being brought back from being stale/disabled for a long time +// XFAIL: * // This test ensures that subsequent invocations of the dependency scanner that re-use previous cache state do not re-use cache entries that contain modules found outside of the current scanner invocation's search paths.