diff --git a/include/swift/AST/DiagnosticsCommon.def b/include/swift/AST/DiagnosticsCommon.def index 696485c7c51ad..476539a15697c 100644 --- a/include/swift/AST/DiagnosticsCommon.def +++ b/include/swift/AST/DiagnosticsCommon.def @@ -32,6 +32,9 @@ ERROR(error_opening_output,none, ERROR(error_closing_output,none, "error closing '%0' for output: %1", (StringRef, StringRef)) +ERROR(error_mccas,none, + "error trying to materialize MCCAS object file: %0", (StringRef)) + ERROR(cannot_find_group_info_file,none, "cannot find group info file at path: '%0'", (StringRef)) diff --git a/include/swift/Frontend/CASOutputBackends.h b/include/swift/Frontend/CASOutputBackends.h index 0ffba2624f4c0..98c9fa1e696bf 100644 --- a/include/swift/Frontend/CASOutputBackends.h +++ b/include/swift/Frontend/CASOutputBackends.h @@ -49,6 +49,10 @@ class SwiftCASOutputBackend : public llvm::vfs::OutputBackend { llvm::Error storeCachedDiagnostics(unsigned InputIndex, llvm::StringRef Bytes); + /// Store the MCCAS CASID \p ID as the object file output for the input + /// that corresponds to the \p OutputFilename + llvm::Error storeMCCASObjectID(StringRef OutputFilename, llvm::cas::CASID ID); + private: class Implementation; Implementation &Impl; diff --git a/include/swift/Frontend/CachingUtils.h b/include/swift/Frontend/CachingUtils.h index 25a865594a002..47b7860d066e6 100644 --- a/include/swift/Frontend/CachingUtils.h +++ b/include/swift/Frontend/CachingUtils.h @@ -44,7 +44,7 @@ bool replayCachedCompilerOutputs( llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache, llvm::cas::ObjectRef BaseKey, DiagnosticEngine &Diag, const FrontendInputsAndOutputs &InputsAndOutputs, - CachingDiagnosticsProcessor &CDP, bool CacheRemarks); + CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend); /// Load the cached compile result from cache. std::unique_ptr loadCachedCompileResultFromCacheKey( diff --git a/lib/Frontend/CASOutputBackends.cpp b/lib/Frontend/CASOutputBackends.cpp index dd6bdd7ac9290..ea058edc13345 100644 --- a/lib/Frontend/CASOutputBackends.cpp +++ b/lib/Frontend/CASOutputBackends.cpp @@ -141,6 +141,21 @@ Error SwiftCASOutputBackend::storeCachedDiagnostics(unsigned InputIndex, file_types::ID::TY_CachedDiagnostics); } +Error SwiftCASOutputBackend::storeMCCASObjectID(StringRef OutputFilename, + llvm::cas::CASID ID) { + auto Input = Impl.OutputToInputMap.find(OutputFilename); + if (Input == Impl.OutputToInputMap.end()) + return llvm::createStringError("InputIndex for output file not found!"); + auto InputIndex = Input->second.first; + auto MCRef = Impl.CAS.getReference(ID); + if (!MCRef) + return createStringError("Invalid CASID: " + ID.toString() + + ". No associated ObjectRef found!"); + + Impl.OutputRefs[InputIndex].insert({file_types::TY_Object, *MCRef}); + return Impl.finalizeCacheKeysFor(InputIndex); +} + void SwiftCASOutputBackend::Implementation::initBackend( const FrontendInputsAndOutputs &InputsAndOutputs) { // FIXME: The output to input map might not be enough for example all the diff --git a/lib/Frontend/CachingUtils.cpp b/lib/Frontend/CachingUtils.cpp index aa2faee30b600..69f94cec9d759 100644 --- a/lib/Frontend/CachingUtils.cpp +++ b/lib/Frontend/CachingUtils.cpp @@ -29,6 +29,7 @@ #include "llvm/CAS/HierarchicalTreeBuilder.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/CAS/TreeEntry.h" +#include "llvm/MCCAS/MCCASObjectV1.h" #include "llvm/Option/ArgList.h" #include "llvm/Option/OptTable.h" #include "llvm/Support/Debug.h" @@ -134,7 +135,7 @@ lookupCacheKey(ObjectStore &CAS, ActionCache &Cache, ObjectRef CacheKey) { bool replayCachedCompilerOutputs( ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey, DiagnosticEngine &Diag, const FrontendInputsAndOutputs &InputsAndOutputs, - CachingDiagnosticsProcessor &CDP, bool CacheRemarks) { + CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend) { bool CanReplayAllOutput = true; struct OutputEntry { std::string Path; @@ -143,6 +144,7 @@ bool replayCachedCompilerOutputs( }; SmallVector OutputProxies; std::optional DiagnosticsOutput; + std::string ObjFile; auto replayOutputsForInputFile = [&](const std::string &InputPath, unsigned InputIndex, @@ -176,7 +178,6 @@ bool replayCachedCompilerOutputs( CachedResultLoader Loader(CAS, **OutputRef); LLVM_DEBUG(llvm::dbgs() << "DEBUG: lookup cache key \'" << OutID.toString() << "\' for input \'" << InputPath << "\n";); - if (auto Err = Loader.replay([&](file_types::ID Kind, ObjectRef Ref) -> Error { auto OutputPath = Outputs.find(Kind); @@ -188,6 +189,9 @@ bool replayCachedCompilerOutputs( if (!Proxy) return Proxy.takeError(); + if (Kind == file_types::ID::TY_Object && UseCASBackend) + ObjFile = OutputPath->second; + if (Kind == file_types::ID::TY_CachedDiagnostics) { assert(!DiagnosticsOutput && "more than 1 diagnotics found"); DiagnosticsOutput = OutputEntry{OutputPath->second, OutID, *Proxy}; @@ -281,7 +285,14 @@ bool replayCachedCompilerOutputs( toString(File.takeError())); continue; } - *File << Output.Proxy.getData(); + + if (UseCASBackend && Output.Path == ObjFile) { + auto Schema = std::make_unique(CAS); + if (auto E = Schema->serializeObjectFile(Output.Proxy, *File)) + Diag.diagnose(SourceLoc(), diag::error_mccas, toString(std::move(E))); + } else + *File << Output.Proxy.getData(); + if (auto E = File->keep()) { Diag.diagnose(SourceLoc(), diag::error_closing_output, Output.Path, toString(std::move(E))); diff --git a/lib/Frontend/Frontend.cpp b/lib/Frontend/Frontend.cpp index a9e5c7b7c9ddc..fb4374dd39c14 100644 --- a/lib/Frontend/Frontend.cpp +++ b/lib/Frontend/Frontend.cpp @@ -474,10 +474,31 @@ void CompilerInstance::setupOutputBackend() { // Mirror the output into CAS. if (supportCaching()) { + auto &InAndOuts = Invocation.getFrontendOptions().InputsAndOutputs; CASOutputBackend = createSwiftCachingOutputBackend( - *CAS, *ResultCache, *CompileJobBaseKey, - Invocation.getFrontendOptions().InputsAndOutputs, + *CAS, *ResultCache, *CompileJobBaseKey, InAndOuts, Invocation.getFrontendOptions().RequestedAction); + + if (Invocation.getIRGenOptions().UseCASBackend) { + auto OutputFiles = InAndOuts.copyOutputFilenames(); + std::unordered_set OutputFileSet( + std::make_move_iterator(OutputFiles.begin()), + std::make_move_iterator(OutputFiles.end())); + // Filter the object file output if MCCAS is enabled, we do not want to + // store the object file itself, but store the MCCAS CASID instead. + auto FilterBackend = llvm::vfs::makeFilteringOutputBackend( + CASOutputBackend, + [&, OutputFileSet](StringRef Path, + std::optional Config) { + if (InAndOuts.getPrincipalOutputType() != file_types::ID::TY_Object) + return true; + return !(OutputFileSet.find(Path.str()) != OutputFileSet.end()); + }); + OutputBackend = + llvm::vfs::makeMirroringOutputBackend(OutputBackend, FilterBackend); + return; + } + OutputBackend = llvm::vfs::makeMirroringOutputBackend(OutputBackend, CASOutputBackend); } diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp index ffb44e70c60d0..0804591b90578 100644 --- a/lib/FrontendTool/FrontendTool.cpp +++ b/lib/FrontendTool/FrontendTool.cpp @@ -1478,7 +1478,8 @@ static bool tryReplayCompilerResults(CompilerInstance &Instance) { Instance.getObjectStore(), Instance.getActionCache(), *Instance.getCompilerBaseKey(), Instance.getDiags(), Instance.getInvocation().getFrontendOptions().InputsAndOutputs, *CDP, - Instance.getInvocation().getCASOptions().EnableCachingRemarks); + Instance.getInvocation().getCASOptions().EnableCachingRemarks, + Instance.getInvocation().getIRGenOptions().UseCASBackend); // If we didn't replay successfully, re-start capture. if (!replayed) @@ -1735,6 +1736,18 @@ static bool generateCode(CompilerInstance &Instance, StringRef OutputFilename, createTargetMachine(opts, Instance.getASTContext()); TargetMachine->Options.MCOptions.CAS = Instance.getSharedCASInstance(); + + if (Instance.getInvocation().getCASOptions().EnableCaching && + opts.UseCASBackend) + TargetMachine->Options.MCOptions.ResultCallBack = + [&](const llvm::cas::CASID &ID) -> llvm::Error { + if (auto Err = Instance.getCASOutputBackend().storeMCCASObjectID( + OutputFilename, ID)) + return Err; + + return llvm::Error::success(); + }; + // Free up some compiler resources now that we have an IRModule. freeASTContextIfPossible(Instance); diff --git a/test/CAS/cache_replay_mccas.swift b/test/CAS/cache_replay_mccas.swift new file mode 100644 index 0000000000000..bdc0f0f3cd09a --- /dev/null +++ b/test/CAS/cache_replay_mccas.swift @@ -0,0 +1,69 @@ +// REQUIRES: OS=macosx +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: %t/test.swift -I %t -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas + +/// Check clang module +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd +// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s +// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-HIT-CLANG %s + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd +// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd +// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd +// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd + +/// Run the command first time, expect cache miss. +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \ +// RUN: -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s +// RUN: test -f %t/test.o +// RUN: test -f %t/test.d +// RUN: test -f %t/Test.swiftmodule + +/// Expect cache hit for second time. +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \ +// RUN: -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s +// RUN: test -f %t/test.o +// RUN: test -f %t/test.d +// RUN: test -f %t/Test.swiftmodule + +/// Expect cache miss a subset of outputs. +// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \ +// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s + +/// Cache hit for retry. +// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \ +// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s + +/// Skip cache +// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -cache-disable-replay %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \ +// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --allow-empty --check-prefix=SKIP-CACHE %s + +// CACHE-MISS: remark: cache miss for input +// CACHE-HIT: remark: replay output file '': key 'llvmcas://{{.*}}' +// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}test.o': key 'llvmcas://{{.*}}' +// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}Test.swiftmodule': key 'llvmcas://{{.*}}' +// CACHE-HIT-CLANG: remark: replay output file '': key 'llvmcas://{{.*}}' +// CACHE-HIT-CLANG: remark: replay output file '{{.*}}{{/|\\}}Dummy-{{.*}}.pcm': key 'llvmcas://{{.*}}' +// SKIP-CACHE-NOT: remark: + +//--- test.swift +import Dummy +func testFunc() {} + +//--- module.modulemap +module Dummy { + umbrella header "Dummy.h" +} + +//--- Dummy.h +void dummy(void); diff --git a/test/CAS/cache_replay_multiple_files_mccas.swift b/test/CAS/cache_replay_multiple_files_mccas.swift new file mode 100644 index 0000000000000..9ef67de447de2 --- /dev/null +++ b/test/CAS/cache_replay_multiple_files_mccas.swift @@ -0,0 +1,48 @@ +// REQUIRES: OS=macosx +// RUN: %empty-directory(%t) +// RUN: split-file %s %t + +// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O \ +// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \ +// RUN: %t/test.swift %t/foo.swift -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas + +// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json +// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid + +// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd +// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd + +/// Test compile multiple inputs with batch mode. +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s +// RUN: test -f %t/test.o +// RUN: test -f %t/foo.o +// RUN: test -f %t/Test.swiftmodule + +/// Expect cache hit second time +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s +// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o \ +// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s +// RUN: test -f %t/test.o +// RUN: test -f %t/foo.o +// RUN: test -f %t/Test.swiftmodule + +//--- test.swift +func testFunc() {} + +//--- foo.swift +func foo() {} + +// CACHE-MISS: remark: cache miss for input +// CACHE-MISS-NOT: remark: replay output file +// CACHE-HIT: remark: replay output file +// CACHE-HIT-NOT: remark: cache miss for input