diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index db20e2daa..6ab44c325 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -10,7 +10,6 @@ add_library(SwiftDriver "ExplicitModuleBuilds/ClangVersionedDependencyResolution.swift" "ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift" "ExplicitModuleBuilds/ModuleDependencyScanning.swift" - "ExplicitModuleBuilds/PlaceholderDependencyResolution.swift" "ExplicitModuleBuilds/SerializableModuleArtifacts.swift" "ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift" "ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift" diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index f00966658..e16560770 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -305,12 +305,9 @@ public struct Driver { /// is shared across many targets; otherwise, a new instance is created by the driver itself. @_spi(Testing) public let interModuleDependencyOracle: InterModuleDependencyOracle - // TODO: Once the clients have transitioned to using the InterModuleDependencyOracle API, - // this must convey information about the externally-prebuilt targets only - /// All external artifacts a build system (e.g. SwiftPM) may pass in as input to the explicit - /// build of the current module. Consists of a map of externally-built targets, and a map of all previously - /// discovered/scanned modules and their infos. - @_spi(Testing) public var externalBuildArtifacts: ExternalBuildArtifacts? = nil + /// A dictionary of external targets that are a part of the same build, mapping to filesystem paths + /// of their module files + @_spi(Testing) public var externalTargetModulePathMap: ExternalTargetModulePathMap? = nil /// A collection of all the flags the selected toolchain's `swift-frontend` supports public let supportedFrontendFlags: Set @@ -387,9 +384,14 @@ public struct Driver { /// expand response files, etc. By default this is the local filesystem. /// - Parameter executor: Used by the driver to execute jobs. The default argument /// is present to streamline testing, it shouldn't be used in production. - /// - Parameter externalBuildArtifacts: All external artifacts a build system may pass in as input to the explicit - /// build of the current module. Consists of a map of externally-built targets, and a map of all previously - /// discovered/scanned modules. + /// - Parameter integratedDriver: Used to distinguish whether the driver is being used as + /// an executable or as a library. + /// - Parameter compilerExecutableDir: Directory that contains the compiler executable to be used. + /// Used when in `integratedDriver` mode as a substitute for the driver knowing its executable path. + /// - Parameter externalTargetModulePathMap: A dictionary of external targets that are a part of + /// the same build, mapping to filesystem paths of their module files. + /// - Parameter interModuleDependencyOracle: An oracle for querying inter-module dependencies, + /// shared across different module builds by a build system. public init( args: [String], env: [String: String] = ProcessEnv.vars, @@ -398,9 +400,6 @@ public struct Driver { executor: DriverExecutor, integratedDriver: Bool = true, compilerExecutableDir: AbsolutePath? = nil, - // FIXME: Duplication with externalBuildArtifacts and externalTargetModulePathMap - // is a temporary backwards-compatibility shim to help transition SwiftPM to the new API - externalBuildArtifacts: ExternalBuildArtifacts? = nil, externalTargetModulePathMap: ExternalTargetModulePathMap? = nil, interModuleDependencyOracle: InterModuleDependencyOracle? = nil ) throws { @@ -411,10 +410,8 @@ public struct Driver { self.diagnosticEngine = diagnosticsEngine self.executor = executor - if let externalArtifacts = externalBuildArtifacts { - self.externalBuildArtifacts = externalArtifacts - } else if let externalTargetPaths = externalTargetModulePathMap { - self.externalBuildArtifacts = (externalTargetPaths, [:]) + if let externalTargetPaths = externalTargetModulePathMap { + self.externalTargetModulePathMap = externalTargetPaths } if case .subcommand = try Self.invocationRunMode(forArgs: args).mode { @@ -507,14 +504,6 @@ public struct Driver { self.interModuleDependencyOracle = dependencyOracle } else { self.interModuleDependencyOracle = InterModuleDependencyOracle() - - // This is a shim for backwards-compatibility with ModuleInfoMap-based API - // used by SwiftPM - if let externalArtifacts = externalBuildArtifacts { - if !externalArtifacts.1.isEmpty { - try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1) - } - } } self.fileListThreshold = try Self.computeFileListThreshold(&self.parsedOptions, diagnosticsEngine: diagnosticsEngine) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift index ca9019a79..5040ec1f2 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift @@ -16,12 +16,6 @@ import Foundation /// A map from a module identifier to a path to its .swiftmodule file. public typealias ExternalTargetModulePathMap = [ModuleDependencyId: AbsolutePath] -// FIXME: ExternalBuildArtifacts is a temporary backwards-compatibility shim -// to help transition SwiftPM to the new API. -/// A tuple all external artifacts a build system may pass in as input to the explicit build of the current module -/// Consists of a map of externally-built targets, and a map of all previously discovered/scanned modules. -public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleInfoMap) - /// In Explicit Module Build mode, this planner is responsible for generating and providing /// build jobs for all module dependencies and providing compile command options /// that specify said explicit module dependencies. @@ -444,7 +438,7 @@ extension ExplicitDependencyBuildPlanner { /// Encapsulates some of the common queries of the ExplicitDependencyBuildPlanner with error-checking /// on the dependency graph's structure. -internal extension InterModuleDependencyGraph { +@_spi(Testing) public extension InterModuleDependencyGraph { func moduleInfo(of moduleId: ModuleDependencyId) throws -> ModuleInfo { guard let moduleInfo = modules[moduleId] else { throw Driver.Error.missingModuleDependency(moduleId.moduleName) diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift index 4a2819d74..72d7e3bd7 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/CommonDependencyOperations.swift @@ -15,11 +15,9 @@ import TSCBasic /// For targets that are built alongside the driver's current module, the scanning action will report them as /// textual targets to be built from source. Because we can rely on these targets to have been built prior /// to the driver's current target, we resolve such external targets as prebuilt binary modules, in the graph. - mutating func resolveExternalDependencies(for externalBuildArtifacts: ExternalBuildArtifacts) + mutating func resolveExternalDependencies(for externalTargetModulePathMap: ExternalTargetModulePathMap) throws { - let externalTargetModulePathMap = externalBuildArtifacts.0 - - for (externalModuleId, externalModulePath) in externalTargetModulePathMap { + for (externalModuleId, externalModulePath) in externalTargetModulePathMap { // Replace the occurence of a Swift module to-be-built from source-file // to an info that describes a pre-built binary module. let swiftModuleId: ModuleDependencyId = .swift(externalModuleId.moduleName) @@ -30,57 +28,22 @@ import TSCBasic // a dependency on a target that is not actually used. continue } - - let newModuleId: ModuleDependencyId = .swiftPrebuiltExternal(externalModuleId.moduleName) + + let newModuleId: ModuleDependencyId = .swiftPrebuiltExternal(externalModuleId.moduleName) let newExternalModuleDetails = - try SwiftPrebuiltExternalModuleDetails(compiledModulePath: - TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern())) + try SwiftPrebuiltExternalModuleDetails(compiledModulePath: + TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern())) let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: VirtualPath.absolute(externalModulePath).intern()), sourceFiles: [], directDependencies: currentInfo.directDependencies, details: .swiftPrebuiltExternal(newExternalModuleDetails)) - Self.replaceModule(originalId: swiftModuleId, replacementId: newModuleId, + Self.replaceModule(originalId: swiftModuleId, replacementId: newModuleId, replacementInfo: newInfo, in: &modules) } } } -@_spi(Testing) public extension InterModuleDependencyOracle { - /// An API to allow clients to accumulate InterModuleDependencyGraphs across mutiple main externalModules/targets - /// into a single collection of discovered externalModules. - func mergeModules(from dependencyGraph: InterModuleDependencyGraph) throws { - try queue.sync { - for (moduleId, moduleInfo) in dependencyGraph.modules { - try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &externalModules) - } - } - } - - // This is a backwards-compatibility shim to handle existing ModuleInfoMap-based API - // used by SwiftPM - func mergeModules(from moduleInfoMap: ModuleInfoMap) throws { - try queue.sync { - for (moduleId, moduleInfo) in moduleInfoMap { - try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &externalModules) - } - } - } -} - -public extension InterModuleDependencyGraph { - // This is a shim for backwards-compatibility with existing API used by SwiftPM. - // TODO: After SwiftPM switches to using the oracle, this should be deleted. - static func mergeModules( - from dependencyGraph: InterModuleDependencyGraph, - into discoveredModules: inout ModuleInfoMap - ) throws { - for (moduleId, moduleInfo) in dependencyGraph.modules { - try mergeModule(moduleId, moduleInfo, into: &discoveredModules) - } - } -} - extension InterModuleDependencyGraph { /// Compute a set of modules that are "reachable" (form direct or transitive dependency) /// from each module in the graph. diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index f5e1133b1..e6ad7b303 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -118,44 +118,5 @@ public class InterModuleDependencyOracle { /// A reference to an instance of the compiler's libSwiftScan shared library private var swiftScanLibInstance: SwiftScan? = nil - - // The below API is a legacy implementation of the oracle that is in-place to allow clients to - // transition to the new API. It is to be removed once that transition is complete. - /// The complete set of modules discovered so far, spanning potentially multiple targets, - /// accumulated across builds of multiple targets. - /// TODO: This is currently only used for placeholder resolution. libSwiftScan should allow us to move away - /// from the concept of a placeholder module so we should be able to get rid of this in the future. - internal var externalModules: ModuleInfoMap = [:] - /// Query the ModuleInfo of a module with a given ID - @_spi(Testing) public func getExternalModuleInfo(of moduleId: ModuleDependencyId) -> ModuleInfo? { - queue.sync { - return externalModules[moduleId] - } - } } -// This is a shim for backwards-compatibility with existing API used by SwiftPM. -// TODO: After SwiftPM switches to using the oracle, this should be deleted. -extension Driver { - public var interModuleDependencyGraph: InterModuleDependencyGraph? { - let mainModuleId : ModuleDependencyId = .swift(moduleOutputInfo.name) - var mainModuleDependencyGraph = - InterModuleDependencyGraph(mainModuleName: moduleOutputInfo.name) - - addModule(moduleId: mainModuleId, - moduleInfo: interModuleDependencyOracle.getExternalModuleInfo(of: mainModuleId)!, - into: &mainModuleDependencyGraph) - return mainModuleDependencyGraph - } - - private func addModule(moduleId: ModuleDependencyId, - moduleInfo: ModuleInfo, - into dependencyGraph: inout InterModuleDependencyGraph) { - dependencyGraph.modules[moduleId] = moduleInfo - moduleInfo.directDependencies?.forEach { dependencyId in - addModule(moduleId: dependencyId, - moduleInfo: interModuleDependencyOracle.getExternalModuleInfo(of: dependencyId)!, - into: &dependencyGraph) - } - } -} diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index 38b78efa6..3174155fb 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -52,48 +52,11 @@ public extension Driver { moduleDependencyGraphUse: .dependencyScan) // FIXME: MSVC runtime flags - // Pass in external target dependencies to be treated as placeholder dependencies by the scanner - if let externalBuildArtifacts = externalBuildArtifacts { - let dependencyPlaceholderMapFile = - try serializeExternalDependencyArtifacts(externalBuildArtifacts: externalBuildArtifacts) - commandLine.appendFlag("-placeholder-dependency-module-map-file") - commandLine.appendPath(dependencyPlaceholderMapFile) - } - // Pass on the input files commandLine.append(contentsOf: inputFiles.map { .path($0.file) }) return (inputs, commandLine) } - /// Serialize a map of placeholder (external) dependencies for the dependency scanner. - private func serializeExternalDependencyArtifacts(externalBuildArtifacts: ExternalBuildArtifacts) - throws -> VirtualPath { - let (externalTargetModulePathMap, externalModuleInfoMap) = externalBuildArtifacts - var placeholderArtifacts: [SwiftModuleArtifactInfo] = [] - - // Explicit external targets - for (moduleId, binaryModulePath) in externalTargetModulePathMap { - let modPath = TextualVirtualPath(path: VirtualPath.absolute(binaryModulePath).intern()) - placeholderArtifacts.append( - SwiftModuleArtifactInfo(name: moduleId.moduleName, - modulePath: modPath)) - } - - // All other already-scanned Swift modules - for (moduleId, moduleInfo) in externalModuleInfoMap - where !externalTargetModulePathMap.keys.contains(moduleId) { - guard case .swift(_) = moduleId else { continue } - placeholderArtifacts.append( - SwiftModuleArtifactInfo(name: moduleId.moduleName, - modulePath: moduleInfo.modulePath)) - } - let encoder = JSONEncoder() - encoder.outputFormatting = [.prettyPrinted] - let contents = try encoder.encode(placeholderArtifacts) - return VirtualPath.createUniqueTemporaryFileWithKnownContents(.init("\(moduleOutputInfo.name)-placeholder-modules.json"), - contents) - } - /// Returns false if the lib is available and ready to use private func initSwiftScanLib() throws -> Bool { // If `-nonlib-dependency-scanner` was specified or the libSwiftScan library cannot be found, diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/PlaceholderDependencyResolution.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/PlaceholderDependencyResolution.swift deleted file mode 100644 index 2ef4f8ed3..000000000 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/PlaceholderDependencyResolution.swift +++ /dev/null @@ -1,170 +0,0 @@ -//===--------------- PlaceholderDependencyResolution.swift ----------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2020 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -import TSCBasic -import TSCUtility -import Foundation - -@_spi(Testing) public extension InterModuleDependencyGraph { - // Building a Swift module in Explicit Module Build mode requires passing all of its module - // dependencies as explicit arguments to the build command. - // - // When the driver's clients (build systems) are planning a build that involves multiple - // Swift modules, planning for each individual module may take place before its dependencies - // have been built. This means that the dependency scanning action will not be able to - // discover such modules. In such cases, the clients must provide the driver with information - // about such external dependencies, including the path to where their compiled .swiftmodule - // will be located, once built, and a full inter-module dependency graph for each such dependence. - // - // The driver will pass down the information about such external dependencies to the scanning - // action, which will generate `placeholder` swift modules for them in the resulting dependency - // graph. The driver will then use the complete dependency graph provided by - // the client for each external dependency and use it to "resolve" the dependency's "placeholder" - // module. - // - // Consider an example SwiftPM package with two targets: target B, and target A, where A - // depends on B: - // SwiftPM will process targets in a topological order and “bubble-up” each target’s - // inter-module dependency graph to its dependees. First, SwiftPM will process B, and be - // able to plan its full build because it does not have any target dependencies. Then the - // driver is tasked with planning a build for A. SwiftPM will pass as input to the driver - // the module dependency graph of its target’s dependencies, in this case, just the - // dependency graph of B. The scanning action for module A will contain a placeholder module B, - // which the driver will then resolve using B's full dependency graph provided by the client. - - /// Resolve all placeholder dependencies using external dependency information provided by the client - mutating func resolvePlaceholderDependencies(for externalBuildArtifacts: ExternalBuildArtifacts, - using dependencyOracle: InterModuleDependencyOracle) - throws { - let externalTargetModulePathMap = externalBuildArtifacts.0 - let placeholderFilter : (ModuleDependencyId) -> Bool = { - if case .swiftPlaceholder(_) = $0 { - return true - } - return false - } - var placeholderModules = modules.keys.filter(placeholderFilter) - - // Resolve all target placeholder modules - let placeholderTargetModules = placeholderModules.filter { externalTargetModulePathMap[$0] != nil } - for moduleId in placeholderTargetModules { - guard let placeholderModulePath = externalTargetModulePathMap[moduleId] else { - throw Driver.Error.missingExternalDependency(moduleId.moduleName) - } - try resolveTargetPlaceholder(placeholderId: moduleId, - placeholderPath: placeholderModulePath, - dependencyOracle: dependencyOracle) - } - - // Process remaining placeholders until there are none left - placeholderModules = modules.keys.filter(placeholderFilter) - while !placeholderModules.isEmpty { - let moduleId = placeholderModules.first! - let swiftModuleId = ModuleDependencyId.swift(moduleId.moduleName) - guard let moduleInfo = dependencyOracle.getExternalModuleInfo(of: swiftModuleId) else { - throw Driver.Error.missingExternalDependency(moduleId.moduleName) - } - - // Insert the resolved module, replacing the placeholder. - try Self.mergeModule(swiftModuleId, moduleInfo, into: &modules) - - // Traverse and add all of this external module's dependencies to the current graph. - try resolvePlaceholderModuleDependencies(moduleId: swiftModuleId, - dependencyOracle: dependencyOracle) - - // Update the set of remaining placeholders to resolve - placeholderModules = modules.keys.filter(placeholderFilter) - } - } -} - -fileprivate extension InterModuleDependencyGraph { - /// Resolve a placeholder dependency that is an external target. - mutating func resolveTargetPlaceholder(placeholderId: ModuleDependencyId, - placeholderPath: AbsolutePath, - dependencyOracle: InterModuleDependencyOracle) - throws { - // For this placeholder dependency, generate a new module info containing only the pre-compiled - // module path, and insert it into the current module's dependency graph, - // replacing equivalent placeholder module. - // - // For all dependencies of this placeholder (direct and transitive), insert them - // into this module's graph. - // - Swift dependencies are inserted as-is - // - Clang dependencies are inserted as-is, if a matching Clang module is already found - // in this module's graph, we merge moduleInfos of the two modules, in order to obtain - // a super-set of their dependencies at all possible PCMArgs variants. - // - // The placeholder is resolved into a .swiftPrebuiltExternal module in the dependency graph. - // The placeholder's corresponding module may appear in the externalModuleInfoMap as either - // a .swift module or a .swiftPrebuiltExternal module if it had been resolved earlier - // in the multi-module build planning context. - let swiftModuleId = ModuleDependencyId.swift(placeholderId.moduleName) - let swiftPrebuiltModuleId = ModuleDependencyId.swiftPrebuiltExternal(placeholderId.moduleName) - let externalModuleId: ModuleDependencyId - let externalModuleInfo: ModuleInfo - if let moduleInfo = dependencyOracle.getExternalModuleInfo(of: swiftModuleId) { - externalModuleId = swiftModuleId - externalModuleInfo = moduleInfo - } else if let prebuiltModuleInfo = dependencyOracle.getExternalModuleInfo(of: swiftPrebuiltModuleId) { - externalModuleId = swiftPrebuiltModuleId - externalModuleInfo = prebuiltModuleInfo - } else { - throw Driver.Error.missingExternalDependency(placeholderId.moduleName) - } - - let placeholderHandle = try VirtualPath.intern(path: placeholderPath.pathString) - let newExternalModuleDetails = - try SwiftPrebuiltExternalModuleDetails(compiledModulePath: - TextualVirtualPath(path: placeholderHandle)) - let newInfo = ModuleInfo(modulePath: TextualVirtualPath(path: placeholderHandle), - sourceFiles: [], - directDependencies: externalModuleInfo.directDependencies, - details: .swiftPrebuiltExternal(newExternalModuleDetails)) - - // Insert the resolved module, replacing the placeholder. - try Self.mergeModule(swiftPrebuiltModuleId, newInfo, into: &modules) - - // Traverse and add all of this external target's dependencies to the current graph. - try resolvePlaceholderModuleDependencies(moduleId: externalModuleId, - dependencyOracle: dependencyOracle) - } - - /// Resolve all dependencies of a placeholder module (direct and transitive), but merging them into the current graph. - mutating func resolvePlaceholderModuleDependencies(moduleId: ModuleDependencyId, - dependencyOracle: InterModuleDependencyOracle) - throws { - guard let resolvingModuleInfo = dependencyOracle.getExternalModuleInfo(of: moduleId) else { - throw Driver.Error.missingExternalDependency(moduleId.moduleName) - } - - // Breadth-first traversal of all the dependencies of this module - var visited: Set = [] - var toVisit: [ModuleDependencyId] = resolvingModuleInfo.directDependencies ?? [] - var currentIndex = 0 - while let currentId = toVisit[currentIndex...].first { - currentIndex += 1 - visited.insert(currentId) - guard let currentInfo = dependencyOracle.getExternalModuleInfo(of: currentId) else { - throw Driver.Error.missingExternalDependency(currentId.moduleName) - } - - try Self.mergeModule(currentId, currentInfo, into: &modules) - - let currentDependencies = currentInfo.directDependencies ?? [] - for childId in currentDependencies where !visited.contains(childId) { - if !toVisit.contains(childId) { - toVisit.append(childId) - } - } - } - } -} diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index b549e6e7c..381ace4f0 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -509,17 +509,13 @@ extension Driver { return try explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs() } - mutating func gatherModuleDependencies() + @_spi(Testing) public mutating func gatherModuleDependencies() throws -> InterModuleDependencyGraph { var dependencyGraph = try performDependencyScan() - if externalBuildArtifacts != nil { - // Resolve placeholder dependencies in the dependency graph, if any. - // TODO: Should be deprecated once switched over to libSwiftScan in the clients - try dependencyGraph.resolvePlaceholderDependencies(for: externalBuildArtifacts!, - using: interModuleDependencyOracle) - - try dependencyGraph.resolveExternalDependencies(for: externalBuildArtifacts!) + if let externalTargetPaths = externalTargetModulePathMap { + // Resolve external dependencies in the dependency graph, if any. + try dependencyGraph.resolveExternalDependencies(for: externalTargetPaths) } // Re-scan Clang modules at all the targets they will be built against. @@ -528,9 +524,6 @@ extension Driver { // Set dependency modules' paths to be saved in the module cache. try resolveDependencyModulePaths(dependencyGraph: &dependencyGraph) - // Update the dependency oracle, adding this new dependency graph to its store - try interModuleDependencyOracle.mergeModules(from: dependencyGraph) - return dependencyGraph } @@ -538,16 +531,16 @@ extension Driver { /// if one is present, and for Swift modules to use the context hash in the file name. private mutating func resolveDependencyModulePaths(dependencyGraph: inout InterModuleDependencyGraph) throws { - // For Swift module dependencies, set the output path to include - // the module's context hash - try resolveSwiftDependencyModuleFileNames(dependencyGraph: &dependencyGraph) - // If a module cache path is specified, update all module dependencies // to be output into it. if let moduleCachePath = parsedOptions.getLastArgument(.moduleCachePath)?.asSingle { try resolveDependencyModulePathsRelativeToModuleCache(dependencyGraph: &dependencyGraph, moduleCachePath: moduleCachePath) } + + // For Swift module dependencies, set the output path to include + // the module's context hash + try resolveSwiftDependencyModuleFileNames(dependencyGraph: &dependencyGraph) } /// For Swift module dependencies, set the output path to include the module's context hash diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index c8043b463..4a6175f14 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -21,10 +21,10 @@ import TestUtilities private func checkExplicitModuleBuildJob(job: Job, pcmArgs: [String], moduleId: ModuleDependencyId, - dependencyOracle: InterModuleDependencyOracle, + dependencyGraph: InterModuleDependencyGraph, pcmFileEncoder: (ModuleInfo, [String]) -> VirtualPath.Handle) throws { - let moduleInfo = dependencyOracle.getExternalModuleInfo(of: moduleId)! + let moduleInfo = try dependencyGraph.moduleInfo(of: moduleId) var downstreamPCMArgs = pcmArgs switch moduleInfo.details { case .swift(let swiftModuleDetails): @@ -61,7 +61,7 @@ throws { XCTAssertTrue(job.commandLine.contains(.flag(String("-fno-implicit-modules")))) try checkExplicitModuleBuildJobDependencies(job: job, pcmArgs: downstreamPCMArgs, moduleInfo: moduleInfo, - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } @@ -70,11 +70,11 @@ throws { private func checkExplicitModuleBuildJobDependencies(job: Job, pcmArgs: [String], moduleInfo : ModuleInfo, - dependencyOracle: InterModuleDependencyOracle, + dependencyGraph: InterModuleDependencyGraph, pcmFileEncoder: (ModuleInfo, [String]) -> VirtualPath.Handle ) throws { for dependencyId in moduleInfo.directDependencies! { - let dependencyInfo = dependencyOracle.getExternalModuleInfo(of: dependencyId)! + let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId) switch dependencyInfo.details { case .swift(let swiftDetails): // Load the dependency JSON and verify this dependency was encoded correctly @@ -137,8 +137,8 @@ private func checkExplicitModuleBuildJobDependencies(job: Job, // Ensure all transitive dependencies got added as well. for transitiveDependencyId in dependencyInfo.directDependencies! { try checkExplicitModuleBuildJobDependencies(job: job, pcmArgs: pcmArgs, - moduleInfo: dependencyOracle.getExternalModuleInfo(of: transitiveDependencyId)!, - dependencyOracle: dependencyOracle, + moduleInfo: try dependencyGraph.moduleInfo(of: transitiveDependencyId), + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } @@ -163,14 +163,6 @@ final class ExplicitModuleBuildTests: XCTestCase { try JSONDecoder().decode( InterModuleDependencyGraph.self, from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!) - let dependencyOracle = InterModuleDependencyOracle() - let scanLibPath = try Driver.getScanLibPath(of: driver.toolchain, - hostTriple: driver.hostTriple, - env: ProcessEnv.vars) - let _ = try dependencyOracle - .verifyOrCreateScannerInstance(fileSystem: localFileSystem, - swiftScanLibPath: scanLibPath) - try dependencyOracle.mergeModules(from: moduleDependencyGraph) driver.explicitDependencyBuildPlanner = try ExplicitDependencyBuildPlanner(dependencyGraph: moduleDependencyGraph, toolchain: driver.toolchain) @@ -193,28 +185,28 @@ final class ExplicitModuleBuildTests: XCTestCase { pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs, moduleId: .clang("SwiftShims"), - dependencyOracle: dependencyOracle, + dependencyGraph: moduleDependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "c_simd", with: pcmArgs, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs, moduleId: .clang("c_simd"), - dependencyOracle: dependencyOracle, + dependencyGraph: moduleDependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(RelativePath("Swift.swiftmodule")): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs, moduleId: .swift("Swift"), - dependencyOracle: dependencyOracle, + dependencyGraph: moduleDependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(RelativePath("_Concurrency.swiftmodule")): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs, moduleId: .swift("_Concurrency"), - dependencyOracle: dependencyOracle, + dependencyGraph: moduleDependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(RelativePath("SwiftOnoneSupport.swiftmodule")): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs, moduleId: .swift("SwiftOnoneSupport"), - dependencyOracle: dependencyOracle, + dependencyGraph: moduleDependencyGraph, pcmFileEncoder: pcmFileEncoder) default: XCTFail("Unexpected module dependency build job output: \(job.outputs[0].file)") @@ -223,89 +215,6 @@ final class ExplicitModuleBuildTests: XCTestCase { } } - func testModuleDependencyWithExternalCommandGeneration() throws { - do { - // Construct a faux external dependency input for module B - let inputDependencyGraph = - try JSONDecoder().decode( - InterModuleDependencyGraph.self, - from: ModuleDependenciesInputs.bPlaceHolderInput.data(using: .utf8)!) - let targetModulePathMap: ExternalTargetModulePathMap = - [ModuleDependencyId.swiftPlaceholder("B"):AbsolutePath("/Somewhere/B.swiftmodule")] - - let executor = try SwiftDriverExecutor(diagnosticsEngine: DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]), - processSet: ProcessSet(), - fileSystem: localFileSystem, - env: ProcessEnv.vars) - let dependencyOracle = InterModuleDependencyOracle() - try dependencyOracle.mergeModules(from: inputDependencyGraph) - - // Construct a module dependency graph that will contain .swiftPlaceholder("B"), - // .swiftPlaceholder("Swift"), .swiftPlaceholder("SwiftOnoneSupport") - var moduleDependencyGraph = - try JSONDecoder().decode( - InterModuleDependencyGraph.self, - from: ModuleDependenciesInputs.fastDependencyScannerPlaceholderOutput.data(using: .utf8)!) - - // Construct the driver with explicit external dependency input - let commandLine = ["swiftc", "-experimental-explicit-module-build", - "test.swift", "-module-name", "A", "-g"] - - var driver = try Driver(args: commandLine, executor: executor, - externalBuildArtifacts: (targetModulePathMap, [:]), - interModuleDependencyOracle: dependencyOracle) - let scanLibPath = try Driver.getScanLibPath(of: driver.toolchain, - hostTriple: driver.hostTriple, - env: ProcessEnv.vars) - let _ = try dependencyOracle - .verifyOrCreateScannerInstance(fileSystem: localFileSystem, - swiftScanLibPath: scanLibPath) - - // Plan explicit dependency jobs, after resolving placeholders to actual dependencies. - try moduleDependencyGraph.resolvePlaceholderDependencies(for: (targetModulePathMap, [:]), - using: dependencyOracle) - - // Ensure the graph no longer contains any placeholders - XCTAssertFalse(moduleDependencyGraph.modules.keys.contains { - if case .swiftPlaceholder(_) = $0 { - return true - } - return false - }) - - // Merge the resolved version of the graph into the oracle - try dependencyOracle.mergeModules(from: moduleDependencyGraph) - driver.explicitDependencyBuildPlanner = - try ExplicitDependencyBuildPlanner(dependencyGraph: moduleDependencyGraph, - toolchain: driver.toolchain) - let modulePrebuildJobs = - try driver.explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs() - - XCTAssertEqual(modulePrebuildJobs.count, 2) - let mainModuleJob = try driver.emitModuleJob() - XCTAssertEqual(mainModuleJob.inputs.count, 5) - for input in mainModuleJob.inputs { - switch (input.file) { - case .relative(RelativePath("Swift.swiftmodule")): - continue - case .relative(RelativePath("_Concurrency.swiftmodule")): - continue - case .relative(RelativePath("SwiftOnoneSupport.swiftmodule")): - continue - case .relative(RelativePath("test.swift")): - continue - case .absolute(AbsolutePath("/Somewhere/B.swiftmodule")): - continue - case .temporaryWithKnownContents(_, _): - XCTAssertTrue(matchTemporary(input.file, "A-dependencies.json")) - continue - default: - XCTFail("Unexpected module input: \(input.file)") - } - } - } - } - private func pathMatchesSwiftModule(path: VirtualPath, _ name: String) -> Bool { return path.basenameWithoutExt.starts(with: "\(name)-") && path.extension! == FileType.swiftModule.rawValue @@ -337,9 +246,8 @@ final class ExplicitModuleBuildTests: XCTestCase { let jobs = try driver.planBuild() // Figure out which Triples to use. - let dependencyOracle = driver.interModuleDependencyOracle - let mainModuleInfo = - dependencyOracle.getExternalModuleInfo(of: .swift("testExplicitModuleBuildJobs"))! + let dependencyGraph = try driver.gatherModuleDependencies() + let mainModuleInfo = try dependencyGraph.moduleInfo(of: .swift("testExplicitModuleBuildJobs")) guard case .swift(let mainModuleSwiftDetails) = mainModuleInfo.details else { XCTFail("Main module does not have Swift details field") return @@ -370,27 +278,27 @@ final class ExplicitModuleBuildTests: XCTestCase { outputFilePath.extension! == FileType.swiftModule.rawValue { if pathMatchesSwiftModule(path: outputFilePath, "A") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("A"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "E") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("E"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "G") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("G"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "Swift") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("Swift"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "_Concurrency") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("_Concurrency"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "SwiftOnoneSupport") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("SwiftOnoneSupport"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } // Clang Dependencies @@ -400,54 +308,54 @@ final class ExplicitModuleBuildTests: XCTestCase { case .relative(pcmArgsEncodedRelativeModulePath(for: "A", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("A"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "B", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("B"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "C", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("C"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "G", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("G"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "G", with: pcmArgs9, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs9, moduleId: .clang("G"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) // Module X is a dependency from Clang module "G" discovered only via versioned PCM // re-scan. case .relative(pcmArgsEncodedRelativeModulePath(for: "X", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("X"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "X", with: pcmArgs9, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs9, moduleId: .clang("X"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "SwiftShims", with: pcmArgs9, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs9, moduleId: .clang("SwiftShims"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "SwiftShims", with: pcmArgs15, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs15, moduleId: .clang("SwiftShims"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "SwiftShims", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("SwiftShims"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) default: XCTFail("Unexpected module dependency build job output: \(outputFilePath)") @@ -500,9 +408,8 @@ final class ExplicitModuleBuildTests: XCTestCase { XCTAssertTrue(interpretJob.commandLine.contains(subsequence: ["-Xcc", "-Xclang", "-Xcc", "-fno-implicit-modules"])) // Figure out which Triples to use. - let dependencyOracle = driver.interModuleDependencyOracle - let mainModuleInfo = - dependencyOracle.getExternalModuleInfo(of: .swift("testExplicitModuleBuildJobs"))! + let dependencyGraph = try driver.gatherModuleDependencies() + let mainModuleInfo = try dependencyGraph.moduleInfo(of: .swift("testExplicitModuleBuildJobs")) guard case .swift(let mainModuleSwiftDetails) = mainModuleInfo.details else { XCTFail("Main module does not have Swift details field") return @@ -532,19 +439,19 @@ final class ExplicitModuleBuildTests: XCTestCase { outputFilePath.extension! == FileType.swiftModule.rawValue { if pathMatchesSwiftModule(path: outputFilePath, "A") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("A"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "Swift") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("Swift"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "_Concurrency") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("_Concurrency"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } else if pathMatchesSwiftModule(path: outputFilePath, "SwiftOnoneSupport") { try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .swift("SwiftOnoneSupport"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) } // Clang Dependencies @@ -554,32 +461,32 @@ final class ExplicitModuleBuildTests: XCTestCase { case .relative(pcmArgsEncodedRelativeModulePath(for: "A", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("A"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "B", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("B"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "C", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("C"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "SwiftShims", with: pcmArgs9, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs9, moduleId: .clang("SwiftShims"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "SwiftShims", with: pcmArgs15, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgs15, moduleId: .clang("SwiftShims"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) case .relative(pcmArgsEncodedRelativeModulePath(for: "SwiftShims", with: pcmArgsCurrent, pcmModuleNameEncoder: pcmModuleNameEncoder)): try checkExplicitModuleBuildJob(job: job, pcmArgs: pcmArgsCurrent, moduleId: .clang("SwiftShims"), - dependencyOracle: dependencyOracle, + dependencyGraph: dependencyGraph, pcmFileEncoder: pcmFileEncoder) default: XCTFail("Unexpected module dependency build job output: \(outputFilePath)") @@ -875,32 +782,6 @@ final class ExplicitModuleBuildTests: XCTestCase { } } - func testDependencyGraphMerge() throws { - let moduleDependencyGraph1 = - try JSONDecoder().decode( - InterModuleDependencyGraph.self, - from: ModuleDependenciesInputs.mergeGraphInput1.data(using: .utf8)!) - let moduleDependencyGraph2 = - try JSONDecoder().decode( - InterModuleDependencyGraph.self, - from: ModuleDependenciesInputs.mergeGraphInput2.data(using: .utf8)!) - - var accumulatingModuleInfoMap: [ModuleDependencyId: ModuleInfo] = [:] - - try InterModuleDependencyGraph.mergeModules(from: moduleDependencyGraph1, - into: &accumulatingModuleInfoMap) - try InterModuleDependencyGraph.mergeModules(from: moduleDependencyGraph2, - into: &accumulatingModuleInfoMap) - - // Ensure the dependencies of the diplicate clang "B" module are merged - let clangIDs = accumulatingModuleInfoMap.keys.filter { $0.moduleName == "B" } - XCTAssertTrue(clangIDs.count == 1) - let clangBInfo = accumulatingModuleInfoMap[clangIDs[0]]! - XCTAssertTrue(clangBInfo.directDependencies!.count == 2) - XCTAssertTrue(clangBInfo.directDependencies!.contains(ModuleDependencyId.clang("D"))) - XCTAssertTrue(clangBInfo.directDependencies!.contains(ModuleDependencyId.clang("C"))) - } - func testExplicitSwiftModuleMap() throws { let jsonExample : String = """ [