Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions include/swift/AST/DiagnosticsCommon.def
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ ERROR(error_opening_output,none,
ERROR(error_closing_output,none,
"error closing '%0' for output: %1", (StringRef, StringRef))

ERROR(error_mccas,none,
"error trying to materialize MCCAS object file: %0", (StringRef))

ERROR(cannot_find_group_info_file,none,
"cannot find group info file at path: '%0'", (StringRef))

Expand Down
4 changes: 4 additions & 0 deletions include/swift/Frontend/CASOutputBackends.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ class SwiftCASOutputBackend : public llvm::vfs::OutputBackend {
llvm::Error storeCachedDiagnostics(unsigned InputIndex,
llvm::StringRef Bytes);

/// Store the MCCAS CASID \p ID as the object file output for the input
/// that corresponds to the \p OutputFilename
llvm::Error storeMCCASObjectID(StringRef OutputFilename, llvm::cas::CASID ID);

private:
class Implementation;
Implementation &Impl;
Expand Down
2 changes: 1 addition & 1 deletion include/swift/Frontend/CachingUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ bool replayCachedCompilerOutputs(
llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
llvm::cas::ObjectRef BaseKey, DiagnosticEngine &Diag,
const FrontendInputsAndOutputs &InputsAndOutputs,
CachingDiagnosticsProcessor &CDP, bool CacheRemarks);
CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend);

/// Load the cached compile result from cache.
std::unique_ptr<llvm::MemoryBuffer> loadCachedCompileResultFromCacheKey(
Expand Down
15 changes: 15 additions & 0 deletions lib/Frontend/CASOutputBackends.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,21 @@ Error SwiftCASOutputBackend::storeCachedDiagnostics(unsigned InputIndex,
file_types::ID::TY_CachedDiagnostics);
}

Error SwiftCASOutputBackend::storeMCCASObjectID(StringRef OutputFilename,
llvm::cas::CASID ID) {
auto Input = Impl.OutputToInputMap.find(OutputFilename);
if (Input == Impl.OutputToInputMap.end())
return llvm::createStringError("InputIndex for output file not found!");
auto InputIndex = Input->second.first;
auto MCRef = Impl.CAS.getReference(ID);
if (!MCRef)
return createStringError("Invalid CASID: " + ID.toString() +
". No associated ObjectRef found!");

Impl.OutputRefs[InputIndex].insert({file_types::TY_Object, *MCRef});
return Impl.finalizeCacheKeysFor(InputIndex);
}

void SwiftCASOutputBackend::Implementation::initBackend(
const FrontendInputsAndOutputs &InputsAndOutputs) {
// FIXME: The output to input map might not be enough for example all the
Expand Down
17 changes: 14 additions & 3 deletions lib/Frontend/CachingUtils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include "llvm/CAS/HierarchicalTreeBuilder.h"
#include "llvm/CAS/ObjectStore.h"
#include "llvm/CAS/TreeEntry.h"
#include "llvm/MCCAS/MCCASObjectV1.h"
#include "llvm/Option/ArgList.h"
#include "llvm/Option/OptTable.h"
#include "llvm/Support/Debug.h"
Expand Down Expand Up @@ -134,7 +135,7 @@ lookupCacheKey(ObjectStore &CAS, ActionCache &Cache, ObjectRef CacheKey) {
bool replayCachedCompilerOutputs(
ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
DiagnosticEngine &Diag, const FrontendInputsAndOutputs &InputsAndOutputs,
CachingDiagnosticsProcessor &CDP, bool CacheRemarks) {
CachingDiagnosticsProcessor &CDP, bool CacheRemarks, bool UseCASBackend) {
bool CanReplayAllOutput = true;
struct OutputEntry {
std::string Path;
Expand All @@ -143,6 +144,7 @@ bool replayCachedCompilerOutputs(
};
SmallVector<OutputEntry> OutputProxies;
std::optional<OutputEntry> DiagnosticsOutput;
std::string ObjFile;

auto replayOutputsForInputFile = [&](const std::string &InputPath,
unsigned InputIndex,
Expand Down Expand Up @@ -176,7 +178,6 @@ bool replayCachedCompilerOutputs(
CachedResultLoader Loader(CAS, **OutputRef);
LLVM_DEBUG(llvm::dbgs() << "DEBUG: lookup cache key \'" << OutID.toString()
<< "\' for input \'" << InputPath << "\n";);

if (auto Err = Loader.replay([&](file_types::ID Kind,
ObjectRef Ref) -> Error {
auto OutputPath = Outputs.find(Kind);
Expand All @@ -188,6 +189,9 @@ bool replayCachedCompilerOutputs(
if (!Proxy)
return Proxy.takeError();

if (Kind == file_types::ID::TY_Object && UseCASBackend)
ObjFile = OutputPath->second;

if (Kind == file_types::ID::TY_CachedDiagnostics) {
assert(!DiagnosticsOutput && "more than 1 diagnotics found");
DiagnosticsOutput = OutputEntry{OutputPath->second, OutID, *Proxy};
Expand Down Expand Up @@ -281,7 +285,14 @@ bool replayCachedCompilerOutputs(
toString(File.takeError()));
continue;
}
*File << Output.Proxy.getData();

if (UseCASBackend && Output.Path == ObjFile) {
auto Schema = std::make_unique<llvm::mccasformats::v1::MCSchema>(CAS);
if (auto E = Schema->serializeObjectFile(Output.Proxy, *File))
Diag.diagnose(SourceLoc(), diag::error_mccas, toString(std::move(E)));
} else
*File << Output.Proxy.getData();

if (auto E = File->keep()) {
Diag.diagnose(SourceLoc(), diag::error_closing_output, Output.Path,
toString(std::move(E)));
Expand Down
25 changes: 23 additions & 2 deletions lib/Frontend/Frontend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -474,10 +474,31 @@ void CompilerInstance::setupOutputBackend() {

// Mirror the output into CAS.
if (supportCaching()) {
auto &InAndOuts = Invocation.getFrontendOptions().InputsAndOutputs;
CASOutputBackend = createSwiftCachingOutputBackend(
*CAS, *ResultCache, *CompileJobBaseKey,
Invocation.getFrontendOptions().InputsAndOutputs,
*CAS, *ResultCache, *CompileJobBaseKey, InAndOuts,
Invocation.getFrontendOptions().RequestedAction);

if (Invocation.getIRGenOptions().UseCASBackend) {
auto OutputFiles = InAndOuts.copyOutputFilenames();
std::unordered_set<std::string> OutputFileSet(
std::make_move_iterator(OutputFiles.begin()),
std::make_move_iterator(OutputFiles.end()));
// Filter the object file output if MCCAS is enabled, we do not want to
// store the object file itself, but store the MCCAS CASID instead.
auto FilterBackend = llvm::vfs::makeFilteringOutputBackend(
CASOutputBackend,
[&, OutputFileSet](StringRef Path,
std::optional<llvm::vfs::OutputConfig> Config) {
if (InAndOuts.getPrincipalOutputType() != file_types::ID::TY_Object)
return true;
return !(OutputFileSet.find(Path.str()) != OutputFileSet.end());
});
OutputBackend =
llvm::vfs::makeMirroringOutputBackend(OutputBackend, FilterBackend);
return;
}

OutputBackend =
llvm::vfs::makeMirroringOutputBackend(OutputBackend, CASOutputBackend);
}
Expand Down
15 changes: 14 additions & 1 deletion lib/FrontendTool/FrontendTool.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1478,7 +1478,8 @@ static bool tryReplayCompilerResults(CompilerInstance &Instance) {
Instance.getObjectStore(), Instance.getActionCache(),
*Instance.getCompilerBaseKey(), Instance.getDiags(),
Instance.getInvocation().getFrontendOptions().InputsAndOutputs, *CDP,
Instance.getInvocation().getCASOptions().EnableCachingRemarks);
Instance.getInvocation().getCASOptions().EnableCachingRemarks,
Instance.getInvocation().getIRGenOptions().UseCASBackend);

// If we didn't replay successfully, re-start capture.
if (!replayed)
Expand Down Expand Up @@ -1735,6 +1736,18 @@ static bool generateCode(CompilerInstance &Instance, StringRef OutputFilename,
createTargetMachine(opts, Instance.getASTContext());

TargetMachine->Options.MCOptions.CAS = Instance.getSharedCASInstance();

if (Instance.getInvocation().getCASOptions().EnableCaching &&
opts.UseCASBackend)
TargetMachine->Options.MCOptions.ResultCallBack =
[&](const llvm::cas::CASID &ID) -> llvm::Error {
if (auto Err = Instance.getCASOutputBackend().storeMCCASObjectID(
OutputFilename, ID))
return Err;

return llvm::Error::success();
};

// Free up some compiler resources now that we have an IRModule.
freeASTContextIfPossible(Instance);

Expand Down
69 changes: 69 additions & 0 deletions test/CAS/cache_replay_mccas.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// REQUIRES: OS=macosx
// RUN: %empty-directory(%t)
// RUN: split-file %s %t

// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O -module-cache-path %t/clang-module-cache \
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
// RUN: %t/test.swift -I %t -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas

/// Check clang module
// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json clang:Dummy > %t/dummy.cmd
// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
// RUN: %swift_frontend_plain @%t/dummy.cmd -Rcache-compile-job 2>&1 | %FileCheck --check-prefix=CACHE-HIT-CLANG %s

// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid

// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
// RUN: echo "\"-disable-implicit-swift-modules\"" >> %t/MyApp.cmd
// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd
// RUN: echo "\"-explicit-swift-module-map-file\"" >> %t/MyApp.cmd
// RUN: echo "\"@%t/map.casid\"" >> %t/MyApp.cmd

/// Run the command first time, expect cache miss.
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \
// RUN: -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
// RUN: test -f %t/test.o
// RUN: test -f %t/test.d
// RUN: test -f %t/Test.swiftmodule

/// Expect cache hit for second time.
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c -emit-dependencies \
// RUN: -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
// RUN: test -f %t/test.o
// RUN: test -f %t/test.d
// RUN: test -f %t/Test.swiftmodule

/// Expect cache miss a subset of outputs.
// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s

/// Cache hit for retry.
// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s

/// Skip cache
// RUN %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -cache-disable-replay %t/test.swift -O -emit-module -emit-module-path %t/Test.swiftmodule -c \
// RUN -module-name Test -o %t/test.o -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --allow-empty --check-prefix=SKIP-CACHE %s

// CACHE-MISS: remark: cache miss for input
// CACHE-HIT: remark: replay output file '<cached-diagnostics>': key 'llvmcas://{{.*}}'
// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}test.o': key 'llvmcas://{{.*}}'
// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}Test.swiftmodule': key 'llvmcas://{{.*}}'
// CACHE-HIT-CLANG: remark: replay output file '<cached-diagnostics>': key 'llvmcas://{{.*}}'
// CACHE-HIT-CLANG: remark: replay output file '{{.*}}{{/|\\}}Dummy-{{.*}}.pcm': key 'llvmcas://{{.*}}'
// SKIP-CACHE-NOT: remark:

//--- test.swift
import Dummy
func testFunc() {}

//--- module.modulemap
module Dummy {
umbrella header "Dummy.h"
}

//--- Dummy.h
void dummy(void);
48 changes: 48 additions & 0 deletions test/CAS/cache_replay_multiple_files_mccas.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// REQUIRES: OS=macosx
// RUN: %empty-directory(%t)
// RUN: split-file %s %t

// RUN: %target-swift-frontend -scan-dependencies -module-name Test -O \
// RUN: -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib \
// RUN: %t/test.swift %t/foo.swift -o %t/deps.json -cache-compile-job -cas-backend -cas-backend-mode=verify -cas-path %t/cas

// RUN: %{python} %S/Inputs/GenerateExplicitModuleMap.py %t/deps.json > %t/map.json
// RUN: llvm-cas --cas %t/cas --make-blob --data %t/map.json > %t/map.casid

// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/MyApp.cmd
// RUN: echo "\"-disable-implicit-string-processing-module-import\"" >> %t/MyApp.cmd
// RUN: echo "\"-disable-implicit-concurrency-module-import\"" >> %t/MyApp.cmd
// RUN: echo "\"-parse-stdlib\"" >> %t/MyApp.cmd

/// Test compile multiple inputs with batch mode.
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o \
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o \
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-MISS %s
// RUN: test -f %t/test.o
// RUN: test -f %t/foo.o
// RUN: test -f %t/Test.swiftmodule

/// Expect cache hit second time
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift %t/foo.swift -emit-module -o %t/Test.swiftmodule \
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job -primary-file %t/test.swift %t/foo.swift -c -o %t/test.o \
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
// RUN: %target-swift-frontend -cache-compile-job -cas-backend -cas-backend-mode=verify -Rcache-compile-job %t/test.swift -primary-file %t/foo.swift -c -o %t/foo.o \
// RUN: -module-name Test -cas-path %t/cas @%t/MyApp.cmd 2>&1 | %FileCheck --check-prefix=CACHE-HIT %s
// RUN: test -f %t/test.o
// RUN: test -f %t/foo.o
// RUN: test -f %t/Test.swiftmodule

//--- test.swift
func testFunc() {}

//--- foo.swift
func foo() {}

// CACHE-MISS: remark: cache miss for input
// CACHE-MISS-NOT: remark: replay output file
// CACHE-HIT: remark: replay output file
// CACHE-HIT-NOT: remark: cache miss for input