From 576cc8cee1dc30cbaf90e4b66f7724ac5ce2b813 Mon Sep 17 00:00:00 2001 From: Ben Langmuir Date: Wed, 28 Feb 2024 13:44:28 -0800 Subject: [PATCH] [Caching] Use clang to prefix-map -fmodule-file-cache-key paths When prefix mapping paths that are used in clang, ensure we are consistently using the same prefix mapper from clang. This prevents mismatches that could cause modules to fail to load. rdar://123324072 --- include/swift/AST/ModuleDependencies.h | 28 +++++++------ include/swift/ClangImporter/ClangImporter.h | 4 +- .../SerializedModuleDependencyCacheFormat.h | 4 +- .../ClangModuleDependencyScanner.cpp | 41 ++++++++++++++++--- .../ModuleDependencyCacheSerialization.cpp | 29 +++++++------ lib/DependencyScan/ScanDependencies.cpp | 6 +-- .../clang_module_output_symlink.swift | 39 ++++++++++++++++++ 7 files changed, 115 insertions(+), 36 deletions(-) create mode 100644 test/ScanDependencies/clang_module_output_symlink.swift diff --git a/include/swift/AST/ModuleDependencies.h b/include/swift/AST/ModuleDependencies.h index 0ebf7f6d59d96..a83b76b78e048 100644 --- a/include/swift/AST/ModuleDependencies.h +++ b/include/swift/AST/ModuleDependencies.h @@ -366,7 +366,11 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { public: /// Destination output path const std::string pcmOutputPath; - + + /// Same as \c pcmOutputPath, but possibly prefix-mapped using clang's prefix + /// mapper. + const std::string mappedPCMPath; + /// The module map file used to generate the Clang module. const std::string moduleMapFile; @@ -390,6 +394,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { std::string CASClangIncludeTreeRootID; ClangModuleDependencyStorage(const std::string &pcmOutputPath, + const std::string &mappedPCMPath, const std::string &moduleMapFile, const std::string &contextHash, const std::vector &buildCommandLine, @@ -400,9 +405,10 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase { const std::string &moduleCacheKey) : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang, moduleCacheKey), - pcmOutputPath(pcmOutputPath), moduleMapFile(moduleMapFile), - contextHash(contextHash), buildCommandLine(buildCommandLine), - fileDependencies(fileDependencies), capturedPCMArgs(capturedPCMArgs), + pcmOutputPath(pcmOutputPath), mappedPCMPath(mappedPCMPath), + moduleMapFile(moduleMapFile), contextHash(contextHash), + buildCommandLine(buildCommandLine), fileDependencies(fileDependencies), + capturedPCMArgs(capturedPCMArgs), CASFileSystemRootID(CASFileSystemRootID), CASClangIncludeTreeRootID(clangIncludeTreeRoot) {} @@ -526,20 +532,18 @@ class ModuleDependencyInfo { /// Describe the module dependencies for a Clang module that can be /// built from a module map and headers. static ModuleDependencyInfo forClangModule( - const std::string &pcmOutputPath, - const std::string &moduleMapFile, - const std::string &contextHash, + const std::string &pcmOutputPath, const std::string &mappedPCMPath, + const std::string &moduleMapFile, const std::string &contextHash, const std::vector &nonPathCommandLine, const std::vector &fileDependencies, const std::vector &capturedPCMArgs, const std::string &CASFileSystemRootID, const std::string &clangIncludeTreeRoot, const std::string &moduleCacheKey) { - return ModuleDependencyInfo( - std::make_unique( - pcmOutputPath, moduleMapFile, contextHash, - nonPathCommandLine, fileDependencies, capturedPCMArgs, - CASFileSystemRootID, clangIncludeTreeRoot, moduleCacheKey)); + return ModuleDependencyInfo(std::make_unique( + pcmOutputPath, mappedPCMPath, moduleMapFile, contextHash, + nonPathCommandLine, fileDependencies, capturedPCMArgs, + CASFileSystemRootID, clangIncludeTreeRoot, moduleCacheKey)); } /// Describe a placeholder dependency swift module. diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h index df75a57657fb7..90360edbbbde1 100644 --- a/include/swift/ClangImporter/ClangImporter.h +++ b/include/swift/ClangImporter/ClangImporter.h @@ -446,7 +446,9 @@ class ClangImporter final : public ClangModuleLoader { void verifyAllModules() override; using RemapPathCallback = llvm::function_ref; - llvm::SmallVector, 1> bridgeClangModuleDependencies( + llvm::SmallVector, 1> + bridgeClangModuleDependencies( + clang::tooling::dependencies::DependencyScanningTool &clangScanningTool, clang::tooling::dependencies::ModuleDepsGraph &clangDependencies, StringRef moduleOutputPath, RemapPathCallback remapPath = nullptr); diff --git a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h index 30b2824816218..7945761d02e49 100644 --- a/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h +++ b/include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h @@ -39,7 +39,8 @@ using llvm::BCVBR; /// Every .moddepcache file begins with these 4 bytes, for easy identification. const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'}; -const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 5; // optionalModuleImports +const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = + 6; // mappedPCMPath /// Increment this on every change. const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 1; @@ -182,6 +183,7 @@ using SwiftPlaceholderModuleDetailsLayout = using ClangModuleDetailsLayout = BCRecordLayout getClangDepScanningInvocationArguments( return commandLineArgs; } +static std::unique_ptr +getClangPrefixMapper(DependencyScanningTool &clangScanningTool, + ModuleDeps &clangModuleDep, + clang::CompilerInvocation &depsInvocation) { + std::unique_ptr Mapper; + if (clangModuleDep.IncludeTreeID) { + Mapper = std::make_unique(); + } else if (clangModuleDep.CASFileSystemRootID) { + assert(clangScanningTool.getCachingFileSystem()); + Mapper = std::make_unique( + clangScanningTool.getCachingFileSystem()); + } + + if (Mapper) + DepscanPrefixMapping::configurePrefixMapper(depsInvocation, *Mapper); + + return Mapper; +} + ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( - clang::tooling::dependencies::ModuleDepsGraph &clangDependencies, - StringRef moduleOutputPath, RemapPathCallback callback) { + clang::tooling::dependencies::DependencyScanningTool &clangScanningTool, + clang::tooling::dependencies::ModuleDepsGraph &clangDependencies, + StringRef moduleOutputPath, RemapPathCallback callback) { const auto &ctx = Impl.SwiftContext; ModuleDependencyVector result; @@ -206,6 +226,10 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( (void)success; assert(success && "clang option from dep scanner round trip failed"); + // Create a prefix mapper that matches clang's configuration. + auto Mapper = + getClangPrefixMapper(clangScanningTool, clangModuleDep, depsInvocation); + // Clear the cache key for module. The module key is computed from clang // invocation, not swift invocation. depsInvocation.getFrontendOpts().ModuleCacheKeys.clear(); @@ -251,10 +275,14 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies( swiftArgs.push_back(IncludeTree); } + std::string mappedPCMPath = pcmPath; + if (Mapper) + Mapper->mapInPlace(mappedPCMPath); + // Module-level dependencies. llvm::StringSet<> alreadyAddedModules; auto dependencies = ModuleDependencyInfo::forClangModule( - pcmPath, clangModuleDep.ClangModuleMapFile, + pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile, clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, capturedPCMArgs, RootID, IncludeTree, /*module-cache-key*/ ""); for (const auto &moduleName : clangModuleDep.ClangModuleDeps) { @@ -414,7 +442,8 @@ ClangImporter::getModuleDependencies(Identifier moduleName, return {}; } - return bridgeClangModuleDependencies(*clangModuleDependencies, + return bridgeClangModuleDependencies(clangScanningTool, + *clangModuleDependencies, moduleOutputPath, [&](StringRef path) { if (mapper) return mapper->mapToString(path); @@ -479,8 +508,8 @@ bool ClangImporter::addBridgingHeaderDependencies( // Record module dependencies for each new module we found. auto bridgedDeps = bridgeClangModuleDependencies( - clangModuleDependencies->ModuleGraph, cache.getModuleOutputPath(), - [&cache](StringRef path) { + clangScanningTool, clangModuleDependencies->ModuleGraph, + cache.getModuleOutputPath(), [&cache](StringRef path) { return cache.getScanService().remapPath(path); }); cache.recordDependencies(bridgedDeps); diff --git a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp index 0bfbeb491b7da..1166c314148d9 100644 --- a/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp +++ b/lib/DependencyScan/ModuleDependencyCacheSerialization.cpp @@ -557,19 +557,20 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi if (!hasCurrentModule) llvm::report_fatal_error("Unexpected CLANG_MODULE_DETAILS_NODE record"); cache.configureForContextHash(getContextHash()); - unsigned pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID, - fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID, - clangIncludeTreeRootID, moduleCacheKeyID; - ClangModuleDetailsLayout::readRecord(Scratch, pcmOutputPathID, moduleMapPathID, - contextHashID, commandLineArrayID, - fileDependenciesArrayID, - capturedPCMArgsArrayID, - CASFileSystemRootID, - clangIncludeTreeRootID, - moduleCacheKeyID); + unsigned pcmOutputPathID, mappedPCMPathID, moduleMapPathID, contextHashID, + commandLineArrayID, fileDependenciesArrayID, capturedPCMArgsArrayID, + CASFileSystemRootID, clangIncludeTreeRootID, moduleCacheKeyID; + ClangModuleDetailsLayout::readRecord( + Scratch, pcmOutputPathID, mappedPCMPathID, moduleMapPathID, + contextHashID, commandLineArrayID, fileDependenciesArrayID, + capturedPCMArgsArrayID, CASFileSystemRootID, clangIncludeTreeRootID, + moduleCacheKeyID); auto pcmOutputPath = getIdentifier(pcmOutputPathID); if (!pcmOutputPath) llvm::report_fatal_error("Bad pcm output path"); + auto mappedPCMPath = getIdentifier(mappedPCMPathID); + if (!mappedPCMPath) + llvm::report_fatal_error("Bad mapped pcm path"); auto moduleMapPath = getIdentifier(moduleMapPathID); if (!moduleMapPath) llvm::report_fatal_error("Bad module map path"); @@ -597,9 +598,9 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi // Form the dependencies storage object auto moduleDep = ModuleDependencyInfo::forClangModule( - *pcmOutputPath, *moduleMapPath, *contextHash, *commandLineArgs, - *fileDependencies, *capturedPCMArgs, *rootFileSystemID, - *clangIncludeTreeRoot, *moduleCacheKey); + *pcmOutputPath, *mappedPCMPath, *moduleMapPath, *contextHash, + *commandLineArgs, *fileDependencies, *capturedPCMArgs, + *rootFileSystemID, *clangIncludeTreeRoot, *moduleCacheKey); // Add dependencies of this module for (const auto &moduleName : *currentModuleImports) @@ -1036,6 +1037,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo( ClangModuleDetailsLayout::emitRecord( Out, ScratchRecord, AbbrCodes[ClangModuleDetailsLayout::Code], getIdentifier(clangDeps->pcmOutputPath), + getIdentifier(clangDeps->mappedPCMPath), getIdentifier(clangDeps->moduleMapFile), getIdentifier(clangDeps->contextHash), getArrayID(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine), @@ -1240,6 +1242,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays( auto clangDeps = dependencyInfo->getAsClangModule(); assert(clangDeps); addIdentifier(clangDeps->pcmOutputPath); + addIdentifier(clangDeps->mappedPCMPath); addIdentifier(clangDeps->moduleMapFile); addIdentifier(clangDeps->contextHash); addStringArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine, diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index fb20d7789d5a8..523e57f95af18 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -285,7 +285,7 @@ static llvm::Error resolveExplicitModuleInputs( if (!resolvingDepInfo.isClangModule()) { commandLine.push_back("-Xcc"); commandLine.push_back("-fmodule-file=" + depModuleID.ModuleName + "=" + - remapPath(clangDepDetails->pcmOutputPath)); + clangDepDetails->mappedPCMPath); if (!instance.getInvocation() .getClangImporterOptions() .UseClangIncludeTree) { @@ -306,7 +306,7 @@ static llvm::Error resolveExplicitModuleInputs( appendXclang(); commandLine.push_back("-fmodule-file-cache-key"); appendXclang(); - commandLine.push_back(remapPath(clangDepDetails->pcmOutputPath)); + commandLine.push_back(clangDepDetails->mappedPCMPath); appendXclang(); commandLine.push_back(clangDepDetails->moduleCacheKey); } @@ -372,7 +372,7 @@ static llvm::Error resolveExplicitModuleInputs( newCommandLine.push_back("-Xcc"); newCommandLine.push_back("-fmodule-file-cache-key"); newCommandLine.push_back("-Xcc"); - newCommandLine.push_back(remapPath(clangDep->pcmOutputPath)); + newCommandLine.push_back(clangDep->mappedPCMPath); newCommandLine.push_back("-Xcc"); newCommandLine.push_back(clangDep->moduleCacheKey); } diff --git a/test/ScanDependencies/clang_module_output_symlink.swift b/test/ScanDependencies/clang_module_output_symlink.swift new file mode 100644 index 0000000000000..571451384fa3b --- /dev/null +++ b/test/ScanDependencies/clang_module_output_symlink.swift @@ -0,0 +1,39 @@ +// Test that after compiling a module to a path containing a symlink we still +// get the same scanner output. + +// REQUIRES: OS=macosx + +// RUN: %empty-directory(%t) +// RUN: split-file %s %t +// RUN: mkdir %t/module-outputs +// RUN: ln -s module-outputs %t/symlink + +// RUN: %target-swift-frontend -scan-dependencies %t/a.swift -o %t/deps.json -module-name A -emit-dependencies -emit-dependencies-path %t/deps.d -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -I %t -module-cache-path %t/symlink -verify -cache-compile-job -cas-path %t/cas +// Check the contents of the JSON output +// RUN: %validate-json %t/deps.json | %FileCheck %s + +// CHECK: "-fmodule-file=C=[[PCM_PATH:.*symlink.*C-.*.pcm]]" +// CHECK: "-fmodule-file-cache-key" +// CHECK-NEXT: "-Xcc" +// CHECK-NEXT: "[[PCM_PATH]]" +// CHECK-NEXT: "-Xcc" +// CHECK-NEXT: "llvmcas:// + +// Emit one of the modules, which will be in the symlinked path. +// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:C > %t/C.cmd +// RUN: %swift_frontend_plain @%t/C.cmd + +// RUN: %target-swift-frontend -scan-dependencies %t/a.swift -o %t/deps2.json -module-name A -emit-dependencies -emit-dependencies-path %t/deps.d -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -I %t -module-cache-path %t/symlink -verify -cache-compile-job -cas-path %t/cas +// RUN: diff -u %t/deps.json %t/deps2.json + +//--- module.modulemap +module B { header "B.h" } +module C { header "C.h" } + +//--- B.h +#include "C.h" + +//--- C.h + +//--- a.swift +import B