From d2d781a58045fa2c32c4cc2d966a15de4bf8f5a3 Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Mon, 12 Jun 2023 14:12:46 -0400 Subject: [PATCH] [Explicit Module Builds] Add support for header dependencies of binary Swift module dependencies When we encounter a pre-built Swift binary module dependency (without an interface file), such module may have been built with a bridging header, which must still be present and is referenced by the binary .swiftmodule as either a .h or, more-likely, a pre-built .pch in a fully-explicit build. Clients must be able to know about such header dependencies in order to be able to import this binary module, because this binary module may be referencing types brought in via its bridging header. The build-system client (swift-driver) will then ensure these header dependencies are fed as inputs to all requiring compilation tasks. This adds support to the driver to query such header dependencies and feed them as inputs to all requiring compilation tasks. --- Sources/CSwiftScan/include/swiftscan_header.h | 2 + .../ExplicitDependencyBuildPlanner.swift | 7 ++ .../InterModuleDependencyGraph.swift | 5 ++ .../InterModuleDependencyOracle.swift | 13 +++- .../SerializableModuleArtifacts.swift | 6 +- .../SwiftScan/DependencyGraphBuilder.swift | 11 ++- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 37 +++++---- .../ExplicitModuleBuildTests.swift | 78 ++++++++++++++++++- 8 files changed, 139 insertions(+), 20 deletions(-) diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 6ee0282e5..1ca4f1307 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -132,6 +132,8 @@ typedef struct { (*swiftscan_swift_binary_detail_get_module_doc_path)(swiftscan_module_details_t); swiftscan_string_ref_t (*swiftscan_swift_binary_detail_get_module_source_info_path)(swiftscan_module_details_t); + swiftscan_string_set_t * + (*swiftscan_swift_binary_detail_get_header_dependencies)(swiftscan_module_details_t); bool (*swiftscan_swift_binary_detail_get_is_framework)(swiftscan_module_details_t); diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift index 638ceffe8..74039b879 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift @@ -234,6 +234,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT for dependencyModule in swiftDependencyArtifacts { inputs.append(TypedVirtualPath(file: dependencyModule.modulePath.path, type: .swiftModule)) + + for headerDep in dependencyModule.prebuiltHeaderDependencyPaths ?? [] { + commandLine.appendFlags(["-Xcc", "-include-pch", "-Xcc"]) + commandLine.appendPath(VirtualPath.lookup(headerDep.path)) + inputs.append(TypedVirtualPath(file: headerDep.path, type: .pch)) + } } // Clang module dependencies are specified on the command line explicitly @@ -301,6 +307,7 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT swiftDependencyArtifacts.append( SwiftModuleArtifactInfo(name: dependencyId.moduleName, modulePath: TextualVirtualPath(path: swiftModulePath.fileHandle), + headerDependencies: prebuiltModuleDetails.headerDependencyPaths, isFramework: isFramework)) case .swiftPlaceholder: fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)") diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift index 93325319a..67946ec22 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyGraph.swift @@ -146,16 +146,21 @@ public struct SwiftPrebuiltExternalModuleDetails: Codable { /// The path to the .swiftSourceInfo file. public var moduleSourceInfoPath: TextualVirtualPath? + /// The paths to the binary module's header dependencies + public var headerDependencyPaths: [TextualVirtualPath]? + /// A flag to indicate whether or not this module is a framework. public var isFramework: Bool? public init(compiledModulePath: TextualVirtualPath, moduleDocPath: TextualVirtualPath? = nil, moduleSourceInfoPath: TextualVirtualPath? = nil, + headerDependencies: [TextualVirtualPath]? = nil, isFramework: Bool) throws { self.compiledModulePath = compiledModulePath self.moduleDocPath = moduleDocPath self.moduleSourceInfoPath = moduleSourceInfoPath + self.headerDependencyPaths = headerDependencies self.isFramework = isFramework } } diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index fc854225b..0c70785c1 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -132,16 +132,23 @@ public class InterModuleDependencyOracle { @_spi(Testing) public func supportsScannerDiagnostics() throws -> Bool { guard let swiftScan = swiftScanLibInstance else { - fatalError("Attempting to reset scanner cache with no scanner instance.") + fatalError("Attempting to query supported scanner API with no scanner instance.") + } + return swiftScan.supportsScannerDiagnostics + } + + @_spi(Testing) public func supportsBinaryModuleHeaderDependencies() throws -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to query supported scanner API with no scanner instance.") } - return swiftScan.supportsScannerDiagnostics() + return swiftScan.supportsBinaryModuleHeaderDependencies } @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") } - guard swiftScan.supportsScannerDiagnostics() else { + guard swiftScan.supportsScannerDiagnostics else { return nil } let diags = try swiftScan.queryScannerDiagnostics() diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift index 214db8183..ae65f097f 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/SerializableModuleArtifacts.swift @@ -24,15 +24,19 @@ public let docPath: TextualVirtualPath? /// The path for the module's .swiftsourceinfo file public let sourceInfoPath: TextualVirtualPath? + /// Header dependencies of this module + public let prebuiltHeaderDependencyPaths: [TextualVirtualPath]? /// A flag to indicate whether this module is a framework public let isFramework: Bool init(name: String, modulePath: TextualVirtualPath, docPath: TextualVirtualPath? = nil, - sourceInfoPath: TextualVirtualPath? = nil, isFramework: Bool = false) { + sourceInfoPath: TextualVirtualPath? = nil, headerDependencies: [TextualVirtualPath]? = nil, + isFramework: Bool = false) { self.moduleName = name self.modulePath = modulePath self.docPath = docPath self.sourceInfoPath = sourceInfoPath + self.prebuiltHeaderDependencyPaths = headerDependencies self.isFramework = isFramework } } diff --git a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift index 531bf628a..99b101dbb 100644 --- a/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift +++ b/Sources/SwiftDriver/SwiftScan/DependencyGraphBuilder.swift @@ -195,7 +195,7 @@ private extension SwiftScan { // Decode all dependencies of this module let swiftOverlayDependencies: [ModuleDependencyId]? - if supportsSeparateSwiftOverlayDependencies(), + if supportsSeparateSwiftOverlayDependencies, let encodedOverlayDepsRef = api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies(moduleDetailsRef) { let encodedOverlayDependencies = try toSwiftStringArray(encodedOverlayDepsRef.pointee) swiftOverlayDependencies = @@ -228,6 +228,14 @@ private extension SwiftScan { try getOptionalPathDetail(from: moduleDetailsRef, using: api.swiftscan_swift_binary_detail_get_module_source_info_path) + let headerDependencies: [TextualVirtualPath]? + if supportsBinaryModuleHeaderDependencies { + headerDependencies = try getOptionalPathArrayDetail(from: moduleDetailsRef, + using: api.swiftscan_swift_binary_detail_get_header_dependencies) + } else { + headerDependencies = nil + } + let isFramework: Bool if hasBinarySwiftModuleIsFramework { isFramework = api.swiftscan_swift_binary_detail_get_is_framework(moduleDetailsRef) @@ -238,6 +246,7 @@ private extension SwiftScan { return try SwiftPrebuiltExternalModuleDetails(compiledModulePath: compiledModulePath, moduleDocPath: moduleDocPath, moduleSourceInfoPath: moduleSourceInfoPath, + headerDependencies: headerDependencies, isFramework: isFramework) } diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 9323de615..f2deead46 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -250,25 +250,20 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_clang_detail_get_captured_pcm_args != nil } - func serializeScannerCache(to path: AbsolutePath) { - api.swiftscan_scanner_cache_serialize(scanner, - path.description.cString(using: String.Encoding.utf8)) + @_spi(Testing) public var supportsBinaryModuleHeaderDependencies : Bool { + return api.swiftscan_swift_binary_detail_get_header_dependencies != nil } - func loadScannerCache(from path: AbsolutePath) -> Bool { - return api.swiftscan_scanner_cache_load(scanner, - path.description.cString(using: String.Encoding.utf8)) + @_spi(Testing) public var supportsStringDispose : Bool { + return api.swiftscan_string_dispose != nil } - func resetScannerCache() { - api.swiftscan_scanner_cache_reset(scanner) - } - @_spi(Testing) public func supportsSeparateSwiftOverlayDependencies() -> Bool { + @_spi(Testing) public var supportsSeparateSwiftOverlayDependencies : Bool { return api.swiftscan_swift_textual_detail_get_swift_overlay_dependencies != nil } - - @_spi(Testing) public func supportsScannerDiagnostics() -> Bool { + + @_spi(Testing) public var supportsScannerDiagnostics : Bool { return api.swiftscan_scanner_diagnostics_query != nil && api.swiftscan_scanner_diagnostics_reset != nil && api.swiftscan_diagnostic_get_message != nil && @@ -276,8 +271,18 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_diagnostics_set_dispose != nil } - @_spi(Testing) public func supportsStringDispose() -> Bool { - return api.swiftscan_string_dispose != nil + func serializeScannerCache(to path: AbsolutePath) { + api.swiftscan_scanner_cache_serialize(scanner, + path.description.cString(using: String.Encoding.utf8)) + } + + func loadScannerCache(from path: AbsolutePath) -> Bool { + return api.swiftscan_scanner_cache_load(scanner, + path.description.cString(using: String.Encoding.utf8)) + } + + func resetScannerCache() { + api.swiftscan_scanner_cache_reset(scanner) } @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] { @@ -427,6 +432,10 @@ private extension swiftscan_functions_t { self.swiftscan_swift_textual_detail_get_swift_overlay_dependencies = try loadOptional("swiftscan_swift_textual_detail_get_swift_overlay_dependencies") + // Header dependencies of binary modules + self.swiftscan_swift_binary_detail_get_header_dependencies = + try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies") + // MARK: Required Methods func loadRequired(_ symbol: String) throws -> T { guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else { diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index 690b4e575..319d0c39d 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -904,7 +904,83 @@ final class ExplicitModuleBuildTests: XCTestCase { XCTAssertTrue(FileManager.default.fileExists(atPath: moduleFooPath)) } } - + + func testExplicitModuleBuildEndToEndWithBinaryHeaderDeps() throws { + try withTemporaryDirectory { path in + try localFileSystem.changeCurrentWorkingDirectory(to: path) + let moduleCachePath = path.appending(component: "ModuleCache") + try localFileSystem.createDirectory(moduleCachePath) + let PCHPath = path.appending(component: "PCH") + try localFileSystem.createDirectory(PCHPath) + let FooInstallPath = path.appending(component: "Foo") + try localFileSystem.createDirectory(FooInstallPath) + let foo = path.appending(component: "foo.swift") + try localFileSystem.writeFileContents(foo) { + $0 <<< "extension Profiler {" + $0 <<< " public static let count: Int = 42" + $0 <<< "}" + } + let fooHeader = path.appending(component: "foo.h") + try localFileSystem.writeFileContents(fooHeader) { + $0 <<< "struct Profiler { void* ptr; };" + } + let main = path.appending(component: "main.swift") + try localFileSystem.writeFileContents(main) { + $0 <<< "import Foo" + } + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + + var fooBuildDriver = try Driver(args: ["swiftc", + "-explicit-module-build", + "-module-cache-path", moduleCachePath.nativePathString(escaped: true), + "-working-directory", path.nativePathString(escaped: true), + foo.nativePathString(escaped: true), + "-emit-module", "-wmo", "-module-name", "Foo", + "-emit-module-path", FooInstallPath.nativePathString(escaped: true), + "-import-objc-header", fooHeader.nativePathString(escaped: true), + "-pch-output-dir", PCHPath.nativePathString(escaped: true), + FooInstallPath.appending(component: "Foo.swiftmodule").nativePathString(escaped: true)] + + sdkArgumentsForTesting, + env: ProcessEnv.vars) + + // Ensure this tooling supports this functionality + let dependencyOracle = InterModuleDependencyOracle() + let scanLibPath = try XCTUnwrap(fooBuildDriver.toolchain.lookupSwiftScanLib()) + guard try dependencyOracle + .verifyOrCreateScannerInstance(fileSystem: localFileSystem, + swiftScanLibPath: scanLibPath) else { + XCTFail("Dependency scanner library not found") + return + } + guard try dependencyOracle.supportsBinaryModuleHeaderDependencies() else { + throw XCTSkip("libSwiftScan does not support binary module header dependencies.") + } + + let fooJobs = try fooBuildDriver.planBuild() + try fooBuildDriver.run(jobs: fooJobs) + XCTAssertFalse(fooBuildDriver.diagnosticEngine.hasErrors) + + var driver = try Driver(args: ["swiftc", + "-I", FooInstallPath.nativePathString(escaped: true), + "-explicit-module-build", "-emit-module", "-emit-module-path", + path.appending(component: "testEMBETEWBHD.swiftmodule").nativePathString(escaped: true), + "-module-cache-path", moduleCachePath.nativePathString(escaped: true), + "-working-directory", path.nativePathString(escaped: true), + main.nativePathString(escaped: true)] + sdkArgumentsForTesting, + env: ProcessEnv.vars) + let jobs = try driver.planBuild() + let compileJob = try XCTUnwrap(jobs.first(where: { $0.description == "Compiling main main.swift" })) + + // Ensure the header dependency of Foo shows up on client compile commands + XCTAssertTrue(compileJob.commandLine.contains(subsequence: [.flag("-Xcc"), + .flag("-include-pch"), + .flag("-Xcc"), + .path(.absolute(PCHPath.appending(component: "foo.pch")))])) + try driver.run(jobs: jobs) + XCTAssertFalse(driver.diagnosticEngine.hasErrors) + } + } + func testExplicitModuleBuildEndToEnd() throws { try withTemporaryDirectory { path in try localFileSystem.changeCurrentWorkingDirectory(to: path)