From bd2b3c7aeb7c0b98de8a3400060c9665b4895aec Mon Sep 17 00:00:00 2001 From: Steven Wu Date: Fri, 17 Oct 2025 10:05:22 -0700 Subject: [PATCH] [Caching] Allow prefix mapping on legacy layout file To support distributed caching for targets that need to use legacy layout file, the path of legacy layout needs to be remapped. Current handling of the legacy layout file is to ingest all layout files to the CAS FileSystem as part of the compiler resource files. But it cannot convey remapped legacy layout file path to the swift-frontend when distributed caching is enabled. In order to properly support path remapping, the legacy layout file is ingested on demand (thus it doesn't need to be ingested for module compilation), and the remapped path is communicated to swift-front via frontend flag. rdar://162793678 --- .../DependencyScan/ModuleDependencyScanner.h | 6 +- .../ModuleDependencyScanner.cpp | 23 ++---- lib/DependencyScan/ScanDependencies.cpp | 21 ++++++ test/CAS/deps_cas_fs.swift | 12 +-- test/CAS/legacy_layout_remap.swift | 73 +++++++++++++++++++ 5 files changed, 110 insertions(+), 25 deletions(-) create mode 100644 test/CAS/legacy_layout_remap.swift diff --git a/include/swift/DependencyScan/ModuleDependencyScanner.h b/include/swift/DependencyScan/ModuleDependencyScanner.h index 49062e3f5525b..6d0c9ebe5f9f5 100644 --- a/include/swift/DependencyScan/ModuleDependencyScanner.h +++ b/include/swift/DependencyScan/ModuleDependencyScanner.h @@ -145,7 +145,11 @@ class SwiftDependencyTracker { const CompilerInvocation &CI); void startTracking(bool includeCommonDeps = true); - void trackFile(const Twine &path); + + /// Track a file with path. + /// \returns true if the file is tracked, false if the file doesn't exist. + bool trackFile(const Twine &path); + llvm::Expected createTreeFromDependencies(); private: diff --git a/lib/DependencyScan/ModuleDependencyScanner.cpp b/lib/DependencyScan/ModuleDependencyScanner.cpp index 4e1f13817d109..7b14f40f946ec 100644 --- a/lib/DependencyScan/ModuleDependencyScanner.cpp +++ b/lib/DependencyScan/ModuleDependencyScanner.cpp @@ -452,20 +452,6 @@ SwiftDependencyTracker::SwiftDependencyTracker( "SDKSettings.json"); addCommonFile(SDKSettingPath); - // Add Legacy layout file. - const std::vector AllSupportedArches = { - "arm64", "arm64e", "x86_64", "i386", - "armv7", "armv7s", "armv7k", "arm64_32"}; - - for (auto RuntimeLibPath : SearchPathOpts.RuntimeLibraryPaths) { - std::error_code EC; - for (auto &Arch : AllSupportedArches) { - SmallString<256> LayoutFile(RuntimeLibPath); - llvm::sys::path::append(LayoutFile, "layouts-" + Arch + ".yaml"); - addCommonFile(LayoutFile); - } - } - // Add VFSOverlay file. for (auto &Overlay: SearchPathOpts.VFSOverlayFiles) addCommonFile(Overlay); @@ -488,20 +474,21 @@ void SwiftDependencyTracker::startTracking(bool includeCommonDeps) { } } -void SwiftDependencyTracker::trackFile(const Twine &path) { +bool SwiftDependencyTracker::trackFile(const Twine &path) { auto file = FS->openFileForRead(path); if (!file) - return; + return false; auto status = (*file)->status(); if (!status) - return; + return false; auto CASFile = dyn_cast(*file); if (!CASFile) - return; + return false; auto fileRef = CASFile->getObjectRefForContent(); std::string realPath = Mapper ? Mapper->mapToString(path.str()) : path.str(); TrackedFiles.try_emplace(realPath, fileRef, (size_t)status->getSize()); + return true; } llvm::Expected diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp index 737238312dd5d..934076ae9961f 100644 --- a/lib/DependencyScan/ScanDependencies.cpp +++ b/lib/DependencyScan/ScanDependencies.cpp @@ -471,6 +471,27 @@ class ExplicitModuleDependencyResolver { [this](const auto &entry) { tracker->trackFile(entry.second.LibraryPath); }); + StringRef readLegacyTypeInfoPath = + instance.getInvocation().getIRGenOptions().ReadLegacyTypeInfoPath; + if (!readLegacyTypeInfoPath.empty()) { + // If legacy layout is specifed, just need to track that file. + if (tracker->trackFile(readLegacyTypeInfoPath)) + commandline.push_back("-read-legacy-type-info-path=" + + scanner.remapPath(readLegacyTypeInfoPath)); + } else { + // Otherwise, search RuntimeLibrary Path for implicit legacy layout. + // Search logic need to match IRGen/GenType.cpp. + const auto &triple = instance.getInvocation().getLangOptions().Target; + auto archName = swift::getMajorArchitectureName(triple); + SmallString<256> legacyLayoutPath(instance.getInvocation() + .getSearchPathOptions() + .RuntimeLibraryPaths[0]); + llvm::sys::path::append(legacyLayoutPath, + "layouts-" + archName + ".yaml"); + if (tracker->trackFile(legacyLayoutPath)) + commandline.push_back("-read-legacy-type-info-path=" + + scanner.remapPath(legacyLayoutPath)); + } auto root = tracker->createTreeFromDependencies(); if (!root) return diagnoseCASFSCreationError(root.takeError()); diff --git a/test/CAS/deps_cas_fs.swift b/test/CAS/deps_cas_fs.swift index c318849503603..e663e22b36a1c 100644 --- a/test/CAS/deps_cas_fs.swift +++ b/test/CAS/deps_cas_fs.swift @@ -5,7 +5,7 @@ // RUN: mkdir -p %t/cas // RUN: mkdir -p %t/resource/macosx -// RUN: cp %S/../IRGen/Inputs/legacy_type_info/a.yaml %t/resource/macosx/layouts-x86_64.yaml +// RUN: cp %S/../IRGen/Inputs/legacy_type_info/a.yaml %t/resource/macosx/layouts-%target-arch.yaml // RUN: %target-swift-frontend -scan-dependencies -module-cache-path %t/clang-module-cache %s -o %t/deps.json -I %S/../ScanDependencies/Inputs/CHeaders -I %S/../ScanDependencies/Inputs/Swift -emit-dependencies -emit-dependencies-path %t/deps.d -import-objc-header %S/../ScanDependencies/Inputs/CHeaders/Bridging.h -swift-version 4 -cache-compile-job -cas-path %t/cas -module-name Test -resource-dir %t/resource -scanner-output-dir %t // Check the contents of the JSON output @@ -19,13 +19,13 @@ // RUN: %{python} %S/Inputs/SwiftDepsExtractor.py %t/deps.json Test casFSRootID > %t/Test_fs.casid // RUN: %cache-tool -cas-path %t/cas -cache-tool-action print-include-tree-list @%t/Test_fs.casid | %FileCheck %s -check-prefix FS_ROOT_TEST -// FS_ROOT_E-DAG: layouts-x86_64.yaml -// FS_ROOT_E-DAG: E.swiftinterface +// FS_ROOT_E-NOT: layouts-{{.*}}.yaml +// FS_ROOT_E: E.swiftinterface -// FS_ROOT_F-DAG: layouts-x86_64.yaml -// FS_ROOT_F-DAG: F.swiftinterface +// FS_ROOT_F-NOT: layouts-{{.*}}.yaml +// FS_ROOT_F: F.swiftinterface -// FS_ROOT_TEST-DAG: layouts-x86_64.yaml +// FS_ROOT_TEST-DAG: layouts-{{.*}}.yaml // FS_ROOT_TEST-DAG: deps_cas_fs.swift import E diff --git a/test/CAS/legacy_layout_remap.swift b/test/CAS/legacy_layout_remap.swift new file mode 100644 index 0000000000000..c9b7775be18c2 --- /dev/null +++ b/test/CAS/legacy_layout_remap.swift @@ -0,0 +1,73 @@ +// REQUIRES: objc_interop +// UNSUPPORTED: swift_only_stable_abi + +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +/// Build legacy module. +// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -emit-module -enable-library-evolution \ +// RUN: -emit-module-path=%t/resilient_struct.swiftmodule -module-name=resilient_struct -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %S/../Inputs/resilient_struct.swift + +/// Scan with legacy layout. +// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -I %t -c -enable-library-evolution -read-legacy-type-info-path=%t/layout.yaml \ +// RUN: -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O -module-load-mode prefer-serialized \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %t/main.swift -o %t/deps.json -swift-version 4 -cache-compile-job -cas-path %t/cas \ +// RUN: -scanner-prefix-map-paths %t /^tmp + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:SwiftShims > %t/shim.cmd +// RUN: %swift_frontend_plain @%t/shim.cmd + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd + +// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -enable-library-evolution \ +// RUN: -cache-compile-job -cas-path %t/cas -O \ +// RUN: -swift-version 4 -disable-implicit-swift-modules \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name Test -explicit-swift-module-map-file @%t/map.casid \ +// RUN: /^tmp/main.swift @%t/MyApp.cmd -c -o %t/main.o + +/// Now do implicit search. +// RUN: mkdir -p %t/resource/macosx +// RUN: cp %t/layout.yaml %t/resource/macosx/layouts-%target-arch.yaml + +// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -I %t -c -enable-library-evolution \ +// RUN: -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O -module-load-mode prefer-serialized \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: %t/main.swift -o %t/deps2.json -swift-version 4 -cache-compile-job -cas-path %t/cas \ +// RUN: -scanner-prefix-map-paths %t /^tmp -resource-dir %t/resource -I %platform-dylib-dir + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps2.json clang:SwiftShims > %t/shim2.cmd +// RUN: %swift_frontend_plain @%t/shim2.cmd + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps2.json > %t/map2.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map2.json > %t/map2.casid +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps2.json Test > %t/MyApp2.cmd + +// RUN: %target-swift-frontend -target %target-pre-stable-abi-triple -enable-library-evolution \ +// RUN: -cache-compile-job -cas-path %t/cas -O \ +// RUN: -swift-version 4 -disable-implicit-swift-modules \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \ +// RUN: -module-name Test -explicit-swift-module-map-file @%t/map2.casid \ +// RUN: /^tmp/main.swift @%t/MyApp2.cmd -c -o %t/main.o + +//--- main.swift +import resilient_struct + +public class ClassWithResilientRef { + var first: ResilientRef? = nil + var second: Int = 0 +} + +//--- layout.yaml +Name: resilient_struct +Decls: + - Name: 16resilient_struct12ResilientRefV + Size: 8 + Alignment: 8 + ExtraInhabitants: 4096