From 754672e299c84e47af78b9e150132d888a4fb591 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Mon, 12 Jun 2023 14:12:46 -0400 Subject: [PATCH] [Explicit Module Builds] Add support for header dependencies of binary Swift module dependencies When we encounter a pre-built Swift binary module dependency (without an interface file), such module may have been built with a bridging header, which must still be present and is referenced by the binary .swiftmodule as either a .h or, more-likely, a pre-built .pch in a fully-explicit build. Clients must be able to know about such header dependencies in order to be able to import this binary module, because this binary module may be referencing types brought in via its bridging header. The build-system client (swift-driver) will then ensure these header dependencies are fed as inputs to all requiring compilation tasks. This adds support to the driver to query such header dependencies and feed them as inputs to all requiring compilation tasks. --- Sources/CSwiftScan/include/swiftscan_header.h | 2 + .../ExplicitDependencyBuildPlanner.swift | 7 ++ .../InterModuleDependencyGraph.swift | 5 ++ .../InterModuleDependencyOracle.swift | 13 +++- .../SerializableModuleArtifacts.swift | 6 +- .../SwiftScan/DependencyGraphBuilder.swift | 11 ++- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 35 +++++---- .../ExplicitModuleBuildTests.swift | 76 +++++++++++++++++++ 8 files changed, 137 insertions(+), 18 deletions(-) diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 474599fb8..7177b0f8b 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -132,6 +132,8 @@ typedef struct { (*swiftscan_swift_binary_detail_get_module_doc_path)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_binary_detail_get_module_source_info_path)(swiftscan_module_details_t); + swiftscan_string_set_t * + (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t); bool (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t); diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift index 638ceffe8..74039b879 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift @@ -234,6 +234,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT for dependencyModule in swiftDependencyArtifacts { inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path, type: .swiftModule)) + + for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] { + commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"]) + commandLine.appendPath(VirtualPath.lookup(headerDep.path)) + inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch)) + } } // Clang module dependencies are specified on the command line explicitly @@ -301,6 +307,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT swiftDependencyArtifacts.append( SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), + headerDependencies: prebuiltModuleDetails.headerDependencyPaths, isFramework: isFramework)) case .swiftPlaceholder: fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)") diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift index 5bd84115a..f0fdce62f 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift @@ -146,16 +146,21 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable { /// The path to the .swiftSourceInfo file. public var moduleSourceInfoPath: TextualVirtualPath? + /// The paths to the binary module's header dependencies + public var headerDependencyPaths: [TextualVirtualPath]? + /// A flag to indicate whether or not this module is a framework. public var isFramework: Bool? public init(compiledModulePath: TextualVirtualPath, moduleDocPath: TextualVirtualPath? = nil, moduleSourceInfoPath: TextualVirtualPath? = nil, + headerDependencies: [TextualVirtualPath]? = nil, isFramework: Bool) throws { self.compiledModulePath = compiledModulePath self.moduleDocPath = moduleDocPath self.moduleSourceInfoPath = moduleSourceInfoPath + self.headerDependencyPaths = headerDependencies self.isFramework = isFramework } } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index 6237dd6f9..470c17ddb 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -132,16 +132,23 @@ public class InterModuleDependencyOracle { @_spi(Testing) public func supportsScannerDiagnostics() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { - fatalError("Attempting to reset scanner cache with no scanner instance.") + fatalError("Attempting to query supported scanner API with no scanner instance.") + } + return swiftScan.supportsScannerDiagnostics + } + + @_spi(Testing) public func supportsBinaryModuleHeaderDependencies() throws -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to query supported scanner API with no scanner instance.") } - return swiftScan.supportsScannerDiagnostics() + return swiftScan.supportsBinaryModuleHeaderDependencies } @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") } - guard swiftScan.supportsScannerDiagnostics() else { + guard swiftScan.supportsScannerDiagnostics else { return nil } let diags = try swiftScan.queryScannerDiagnostics() diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift index 214db8183..ae65f097f 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift @@ -24,15 +24,19 @@ public let docPath: TextualVirtualPath? /// The path for the module's .swiftsourceinfo file public let sourceInfoPath: TextualVirtualPath? + /// Header dependencies of this module + public let prebuiltHeaderDependencyPaths: [TextualVirtualPath]? /// A flag to indicate whether this module is a framework public let isFramework: Bool init(name: String, modulePath: TextualVirtualPath, docPath: TextualVirtualPath? = nil, - sourceInfoPath: TextualVirtualPath? = nil, isFramework: Bool = false) { + sourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil, + isFramework: Bool = false) { self.moduleName = name self.modulePath = modulePath self.docPath = docPath self.sourceInfoPath = sourceInfoPath + self.prebuiltHeaderDependencyPaths = headerDependencies self.isFramework = isFramework } } diff --git a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift index 531bf628a..99b101dbb 100644 --- a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift +++ b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift @@ -195,7 +195,7 @@ private extension SwiftScan { // Decode all dependencies of this module let swiftOverlayDependencies: [ModuleDependencyId]? - if supportsSeparateSwiftOverlayDependencies(), + if supportsSeparateSwiftOverlayDependencies, let encodedOverlayDepsRef = api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies(moduleDetailsRef) { let encodedOverlayDependencies = try toSwiftStringArray(encodedOverlayDepsRef.pointee) swiftOverlayDependencies = @@ -228,6 +228,14 @@ private extension SwiftScan { try getOptionalPathDetail(from: moduleDetailsRef, using: api.swiftscan_swift_binary_detail_get_module_source_info_path) + let headerDependencies: [TextualVirtualPath]? + if supportsBinaryModuleHeaderDependencies { + headerDependencies = try getOptionalPathArrayDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_binary_detail_get_header_dependencies) + } else { + headerDependencies = nil + } + let isFramework: Bool if hasBinarySwiftModuleIsFramework { isFramework = api.swiftscan_swift_binary_detail_get_is_framework(moduleDetailsRef) @@ -238,6 +246,7 @@ private extension SwiftScan { return try SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath, moduleDocPath: moduleDocPath, moduleSourceInfoPath: moduleSourceInfoPath, + headerDependencies: headerDependencies, isFramework: isFramework) } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 77cd15827..9924f2617 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -250,25 +250,20 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_clang_detail_get_captured_pcm_args != nil } - func serializeScannerCache(to path: AbsolutePath) { - api.swiftscan_scanner_cache_serialize(scanner, - path.description.cString(using: String.Encoding.utf8)) + @_spi(Testing) public var supportsBinaryModuleHeaderDependencies : Bool { + return api.swiftscan_swift_binary_detail_get_header_dependencies != nil } - func loadScannerCache(from path: AbsolutePath) -> Bool { - return api.swiftscan_scanner_cache_load(scanner, - path.description.cString(using: String.Encoding.utf8)) + @_spi(Testing) public var supportsStringDispose : Bool { + return api.swiftscan_string_dispose != nil } - func resetScannerCache() { - api.swiftscan_scanner_cache_reset(scanner) - } - @_spi(Testing) public func supportsSeparateSwiftOverlayDependencies() -> Bool { + @_spi(Testing) public var supportsSeparateSwiftOverlayDependencies : Bool { return api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies != nil } - @_spi(Testing) public func supportsScannerDiagnostics() -> Bool { + @_spi(Testing) public var supportsScannerDiagnostics : Bool { return api.swiftscan_scanner_diagnostics_query != nil && api.swiftscan_scanner_diagnostics_reset != nil && api.swiftscan_diagnostic_get_message != nil && @@ -276,8 +271,18 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_diagnostics_set_dispose != nil } - @_spi(Testing) public func supportsStringDispose() -> Bool { - return api.swiftscan_string_dispose != nil + func serializeScannerCache(to path: AbsolutePath) { + api.swiftscan_scanner_cache_serialize(scanner, + path.description.cString(using: String.Encoding.utf8)) + } + + func loadScannerCache(from path: AbsolutePath) -> Bool { + return api.swiftscan_scanner_cache_load(scanner, + path.description.cString(using: String.Encoding.utf8)) + } + + func resetScannerCache() { + api.swiftscan_scanner_cache_reset(scanner) } @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] { @@ -427,6 +432,10 @@ private extension swiftscan_functions_t { self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies = try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies") + // Header dependencies of binary modules + self.swiftscan_swift_binary_detail_get_header_dependencies = + try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies") + // MARK: Required Methods func loadRequired(_ symbol: String) throws -> T { guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else { diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index ced7c97e7..c51dd22a8 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -905,6 +905,82 @@ final class ExplicitModuleBuildTests: XCTestCase { } } + func testExplicitModuleBuildEndToEndWithBinaryHeaderDeps() throws { + try withTemporaryDirectory { path in + try localFileSystem.changeCurrentWorkingDirectory(to: path) + let moduleCachePath = path.appending(component: "ModuleCache") + try localFileSystem.createDirectory(moduleCachePath) + let PCHPath = path.appending(component: "PCH") + try localFileSystem.createDirectory(PCHPath) + let FooInstallPath = path.appending(component: "Foo") + try localFileSystem.createDirectory(FooInstallPath) + let foo = path.appending(component: "foo.swift") + try localFileSystem.writeFileContents(foo) { + $0 <<< "extension Profiler {" + $0 <<< " public static let count: Int = 42" + $0 <<< "}" + } + let fooHeader = path.appending(component: "foo.h") + try localFileSystem.writeFileContents(fooHeader) { + $0 <<< "struct Profiler { void* ptr; };" + } + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main) { + $0 <<< "import Foo" + } + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + var fooBuildDriver = try Driver(args: ["swiftc", + "-explicit-module-build", + "-module-cache-path", moduleCachePath.nativePathString(escaped: true), + "-working-directory", path.nativePathString(escaped: true), + foo.nativePathString(escaped: true), + "-emit-module", "-wmo", "-module-name", "Foo", + "-emit-module-path", FooInstallPath.nativePathString(escaped: true), + "-import-objc-header", fooHeader.nativePathString(escaped: true), + "-pch-output-dir", PCHPath.nativePathString(escaped: true), + FooInstallPath.appending(component: "Foo.swiftmodule").nativePathString(escaped: true)] + + sdkArgumentsForTesting, + env: ProcessEnv.vars) + + // Ensure this tooling supports this functionality + let dependencyOracle = InterModuleDependencyOracle() + let scanLibPath = try XCTUnwrap(fooBuildDriver.toolchain.lookupSwiftScanLib()) + guard try dependencyOracle + .verifyOrCreateScannerInstance(fileSystem: localFileSystem, + swiftScanLibPath: scanLibPath) else { + XCTFail("Dependency scanner library not found") + return + } + guard try dependencyOracle.supportsBinaryModuleHeaderDependencies() else { + throw XCTSkip("libSwiftScan does not support binary module header dependencies.") + } + + let fooJobs = try fooBuildDriver.planBuild() + try fooBuildDriver.run(jobs: fooJobs) + XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors) + + var driver = try Driver(args: ["swiftc", + "-I", FooInstallPath.nativePathString(escaped: true), + "-explicit-module-build", "-emit-module", "-emit-module-path", + path.appending(component: "testEMBETEWBHD.swiftmodule").nativePathString(escaped: true), + "-module-cache-path", moduleCachePath.nativePathString(escaped: true), + "-working-directory", path.nativePathString(escaped: true), + main.nativePathString(escaped: true)] + sdkArgumentsForTesting, + env: ProcessEnv.vars) + let jobs = try driver.planBuild() + let compileJob = try XCTUnwrap(jobs.first(where: { $0.description == "Compiling main main.swift" })) + + // Ensure the header dependency of Foo shows up on client compile commands + XCTAssertTrue(compileJob.commandLine.contains(subsequence: [.flag("-Xcc"), + .flag("-include-pch"), + .flag("-Xcc"), + .path(.absolute(PCHPath.appending(component: "foo.pch")))])) + try driver.run(jobs: jobs) + XCTAssertFalse(driver.diagnosticEngine.hasErrors) + } + } + func testExplicitModuleBuildEndToEnd() throws { try withTemporaryDirectory { path in try localFileSystem.changeCurrentWorkingDirectory(to: path)