From c4c6f090a78068222ff8bf11e2e9410f6e589841 Mon Sep 17 00:00:00 2001 From: Argyrios Kyrtzidis Date: Mon, 1 May 2023 17:05:02 -0700 Subject: [PATCH] [CAS/libclang] Add libclang APIs for handling and replaying cached compilations * Lookup a cached compilation using a compilation cache key * Get information about the compilation outputs and materialize them (bring them in the local CAS) * Replay a cached compilation These allow a client to retrieve the data of a prior cached compilation and replay it without needing to invoke clang as a separate process. (cherry picked from commit 9c0b819bd637193643b1d6fb7bf4bac371230342) --- clang/include/clang-c/CAS.h | 193 +++++++++ clang/include/clang-c/CXErrorCode.h | 20 + clang/include/clang-c/Dependencies.h | 8 + .../include/clang/Frontend/CompileJobCache.h | 6 + .../clang/Frontend/CompileJobCacheResult.h | 2 + clang/lib/Frontend/CompileJobCache.cpp | 80 +++- clang/lib/Frontend/CompileJobCacheResult.cpp | 4 + clang/test/CAS/libclang-replay-job.c | 61 +++ clang/test/ClangScanDeps/error-c-api.cpp | 2 +- clang/test/ClangScanDeps/flags-c-api.cpp | 2 +- .../mllvm-double-option-error-c-api.c | 2 +- .../test/ClangScanDeps/modules-order-c-api.c | 6 +- .../ClangScanDeps/modules-outputs-c-api.c | 12 +- clang/test/ClangScanDeps/simple-c-api.cpp | 2 +- clang/test/Index/Core/scan-deps-by-mod-name.m | 2 +- clang/test/Index/Core/scan-deps-cas.m | 8 +- clang/test/Index/Core/scan-deps-with-diags.m | 2 +- clang/test/Index/Core/scan-deps.m | 4 +- clang/tools/c-index-test/core_main.cpp | 235 +++++++++-- clang/tools/driver/features.json | 3 + clang/tools/libclang/CCAS.cpp | 396 ++++++++++++++++++ clang/tools/libclang/CDependencies.cpp | 5 + clang/tools/libclang/CMakeLists.txt | 1 + clang/tools/libclang/CXError.cpp | 42 ++ clang/tools/libclang/CXError.h | 23 + clang/tools/libclang/libclang.map | 20 + .../libCASPluginTest/libCASPluginTest.cpp | 36 +- 27 files changed, 1116 insertions(+), 61 deletions(-) create mode 100644 clang/test/CAS/libclang-replay-job.c create mode 100644 clang/tools/libclang/CXError.cpp create mode 100644 clang/tools/libclang/CXError.h diff --git a/clang/include/clang-c/CAS.h b/clang/include/clang-c/CAS.h index 12a99c3ea9334..9b12b61aabea1 100644 --- a/clang/include/clang-c/CAS.h +++ b/clang/include/clang-c/CAS.h @@ -20,6 +20,7 @@ #ifndef LLVM_CLANG_C_CAS_H #define LLVM_CLANG_C_CAS_H +#include "clang-c/CXErrorCode.h" #include "clang-c/CXString.h" #include "clang-c/Platform.h" @@ -53,6 +54,23 @@ typedef struct CXOpaqueCASObjectStore *CXCASObjectStore; */ typedef struct CXOpaqueCASActionCache *CXCASActionCache; +typedef struct CXOpaqueCASObject *CXCASObject; + +/** + * Result of \c clang_experimental_cas_getCachedCompilation. + */ +typedef struct CXOpaqueCASCachedCompilation *CXCASCachedCompilation; + +/** + * Result of \c clang_experimental_cas_replayCompilation. + */ +typedef struct CXOpaqueCASReplayResult *CXCASReplayResult; + +/** + * Used for cancelling asynchronous actions. + */ +typedef struct CXOpaqueCASCancellationToken *CXCASCancellationToken; + /** * Create a \c CXCASOptions object. */ @@ -99,6 +117,181 @@ clang_experimental_cas_Databases_create(CXCASOptions Opts, CXString *Error); */ CINDEX_LINKAGE void clang_experimental_cas_Databases_dispose(CXCASDatabases); +/** + * Loads an object using its printed \p CASID. + * + * \param CASID The printed CASID string for the object. + * \param[out] OutError The error object to pass back to client (if any). + * If non-null the object must be disposed using \c clang_Error_dispose. + * + * \returns The resulting object, or null if the object was not found or an + * error occurred. The object should be disposed using + * \c clang_experimental_cas_CASObject_dispose. + */ +CINDEX_LINKAGE CXCASObject clang_experimental_cas_loadObjectByString( + CXCASDatabases, const char *CASID, CXError *OutError); + +/** + * Asynchronous version of \c clang_experimental_cas_loadObjectByString. + * + * \param CASID The printed CASID string for the object. + * \param Ctx opaque value to pass to the callback. + * \param Callback receives a \c CXCASObject, or \c CXError if an error occurred + * or both NULL if the object was not found or the call was cancelled. + * The objects should be disposed with + * \c clang_experimental_cas_CASObject_dispose or \c clang_Error_dispose. + * \param[out] OutToken if non-null receives a \c CXCASCancellationToken that + * can be used to cancel the call using + * \c clang_experimental_cas_CancellationToken_cancel. The object should be + * disposed using \c clang_experimental_cas_CancellationToken_dispose. + */ +CINDEX_LINKAGE void clang_experimental_cas_loadObjectByString_async( + CXCASDatabases, const char *CASID, void *Ctx, + void (*Callback)(void *Ctx, CXCASObject, CXError), + CXCASCancellationToken *OutToken); + +/** + * Dispose of a \c CXCASObject object. + */ +CINDEX_LINKAGE void clang_experimental_cas_CASObject_dispose(CXCASObject); + +/** + * Looks up a cache key and returns the associated set of compilation output IDs + * + * \param CacheKey The printed compilation cache key string. + * \param Globally if true it is a hint to the underlying CAS implementation + * that the lookup is profitable to be done on a distributed caching level, not + * just locally. + * \param[out] OutError The error object to pass back to client (if any). + * If non-null the object must be disposed using \c clang_Error_dispose. + * + * \returns The resulting object, or null if the cache key was not found or an + * error occurred. The object should be disposed using + * \c clang_experimental_cas_CachedCompilation_dispose. + */ +CINDEX_LINKAGE CXCASCachedCompilation +clang_experimental_cas_getCachedCompilation(CXCASDatabases, + const char *CacheKey, bool Globally, + CXError *OutError); + +/** + * Asynchronous version of \c clang_experimental_cas_getCachedCompilation. + * + * \param CacheKey The printed compilation cache key string. + * \param Globally if true it is a hint to the underlying CAS implementation + * that the lookup is profitable to be done on a distributed caching level, not + * just locally. + * \param Ctx opaque value to pass to the callback. + * \param Callback receives a \c CXCASCachedCompilation, or \c CXError if an + * error occurred or both NULL if the object was not found or the call was + * cancelled. The objects should be disposed with + * \c clang_experimental_cas_CachedCompilation_dispose or \c clang_Error_dispose + * \param[out] OutToken if non-null receives a \c CXCASCancellationToken that + * can be used to cancel the call using + * \c clang_experimental_cas_CancellationToken_cancel. The object should be + * disposed using \c clang_experimental_cas_CancellationToken_dispose. + */ +CINDEX_LINKAGE void clang_experimental_cas_getCachedCompilation_async( + CXCASDatabases, const char *CacheKey, bool Globally, void *Ctx, + void (*Callback)(void *Ctx, CXCASCachedCompilation, CXError), + CXCASCancellationToken *OutToken); + +/** + * Dispose of a \c CXCASCachedCompilation object. + */ +CINDEX_LINKAGE void + clang_experimental_cas_CachedCompilation_dispose(CXCASCachedCompilation); + +/** + * \returns number of compilation outputs. + */ +CINDEX_LINKAGE size_t clang_experimental_cas_CachedCompilation_getNumOutputs( + CXCASCachedCompilation); + +/** + * \returns the compilation output name given the index via \p OutputIdx. + */ +CINDEX_LINKAGE CXString clang_experimental_cas_CachedCompilation_getOutputName( + CXCASCachedCompilation, size_t OutputIdx); + +/** + * \returns the compilation output printed CASID given the index via + * \p OutputIdx. + */ +CINDEX_LINKAGE CXString +clang_experimental_cas_CachedCompilation_getOutputCASIDString( + CXCASCachedCompilation, size_t OutputIdx); + +/** + * \returns whether the compilation output data exist in the local CAS given the + * index via \p OutputIdx. + */ +CINDEX_LINKAGE bool +clang_experimental_cas_CachedCompilation_isOutputMaterialized( + CXCASCachedCompilation, size_t OutputIdx); + +/** + * If distributed caching is available it uploads the compilation outputs and + * the association of key <-> outputs to the distributed cache. + * This allows separating the task of computing the compilation outputs and + * storing them in the local cache, from the task of "uploading" them. + * + * \param Ctx opaque value to pass to the callback. + * \param Callback receives a \c CXError if an error occurred. The error will be + * NULL if the call was successful or cancelled. The error should be disposed + * via \c clang_Error_dispose. + * \param[out] OutToken if non-null receives a \c CXCASCancellationToken that + * can be used to cancel the call using + * \c clang_experimental_cas_CancellationToken_cancel. The object should be + * disposed using \c clang_experimental_cas_CancellationToken_dispose. + */ +CINDEX_LINKAGE void clang_experimental_cas_CachedCompilation_makeGlobal( + CXCASCachedCompilation, void *Ctx, void (*Callback)(void *Ctx, CXError), + CXCASCancellationToken *OutToken); + +/** + * Replays a cached compilation by writing the cached outputs to the filesystem + * and/or stderr based on the given compilation arguments. + * + * \param argc number of compilation arguments. + * \param argv array of compilation arguments. + * \param WorkingDirectory working directory to use, can be NULL. + * \param reserved for future use, caller must pass NULL. + * \param[out] OutError The error object to pass back to client (if any). + * If non-null the object must be disposed using \c clang_Error_dispose. + * + * \returns a \c CXCASReplayResult object or NULL if an error occurred or a + * compilation output was not found in the CAS. The object should be disposed + * via \c clang_experimental_cas_ReplayResult_dispose. + */ +CINDEX_LINKAGE CXCASReplayResult clang_experimental_cas_replayCompilation( + CXCASCachedCompilation, int argc, const char *const *argv, + const char *WorkingDirectory, void *reserved, CXError *OutError); + +/** + * Dispose of a \c CXCASReplayResult object. + */ +CINDEX_LINKAGE void + clang_experimental_cas_ReplayResult_dispose(CXCASReplayResult); + +/** + * Get the diagnostic text of a replayed cached compilation. + */ +CINDEX_LINKAGE +CXString clang_experimental_cas_ReplayResult_getStderr(CXCASReplayResult); + +/** + * Cancel an asynchronous CAS-related action. + */ +CINDEX_LINKAGE void + clang_experimental_cas_CancellationToken_cancel(CXCASCancellationToken); + +/** + * Dispose of a \c CXCASCancellationToken object. + */ +CINDEX_LINKAGE void + clang_experimental_cas_CancellationToken_dispose(CXCASCancellationToken); + /** * Dispose of a \c CXCASObjectStore object. */ diff --git a/clang/include/clang-c/CXErrorCode.h b/clang/include/clang-c/CXErrorCode.h index 510f8d6f300ef..e284b2843a7ff 100644 --- a/clang/include/clang-c/CXErrorCode.h +++ b/clang/include/clang-c/CXErrorCode.h @@ -74,6 +74,26 @@ enum CXErrorCode { CXError_RefactoringNameInvalid = 7 }; +/** + * Represents an error with error code and description string. + */ +typedef struct CXOpaqueError *CXError; + +/** + * \returns the error code. + */ +CINDEX_LINKAGE enum CXErrorCode clang_Error_getCode(CXError); + +/** + * \returns the error description string. + */ +CINDEX_LINKAGE const char *clang_Error_getDescription(CXError); + +/** + * Dispose of a \c CXError object. + */ +CINDEX_LINKAGE void clang_Error_dispose(CXError); + LLVM_CLANG_C_EXTERN_C_END #endif diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index 2334495605fe6..374e35dbc23a5 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -205,6 +205,14 @@ CINDEX_LINKAGE void clang_experimental_DependencyScannerServiceOptions_setCASDatabases( CXDependencyScannerServiceOptions Opts, CXCASDatabases); +/** + * Specify the specific CAS options for the scanner to use for the produced + * compiler arguments. + */ +CINDEX_LINKAGE void +clang_experimental_DependencyScannerServiceOptions_setCASOptions( + CXDependencyScannerServiceOptions Opts, CXCASOptions); + /** * Specify a \c CXCASObjectStore in the given options. If an object store and * action cache are available, the scanner will produce cached commands. diff --git a/clang/include/clang/Frontend/CompileJobCache.h b/clang/include/clang/Frontend/CompileJobCache.h index f1488d6b78077..d4eb8109514c6 100644 --- a/clang/include/clang/Frontend/CompileJobCache.h +++ b/clang/include/clang/Frontend/CompileJobCache.h @@ -14,6 +14,7 @@ namespace clang { class CompilerInstance; +class CompilerInvocation; class DiagnosticsEngine; // Manage caching and replay of compile jobs. @@ -80,6 +81,11 @@ class CompileJobCache { /// \returns true if finished successfully. bool finishComputedResult(CompilerInstance &Clang, bool Success); + static llvm::Expected> replayCachedResult( + std::shared_ptr Invok, + const llvm::cas::CASID &CacheKey, + cas::CompileJobCacheResult &CachedResult, SmallVectorImpl &DiagText); + class CachingOutputs; private: diff --git a/clang/include/clang/Frontend/CompileJobCacheResult.h b/clang/include/clang/Frontend/CompileJobCacheResult.h index 9a4616488a5d0..9b532b1105dce 100644 --- a/clang/include/clang/Frontend/CompileJobCacheResult.h +++ b/clang/include/clang/Frontend/CompileJobCacheResult.h @@ -55,6 +55,8 @@ class CompileJobCacheResult : public ObjectProxy { size_t getNumOutputs() const; + Output getOutput(size_t I) const; + /// Retrieves a specific output specified by \p Kind, if it exists. Optional getOutput(OutputKind Kind) const; diff --git a/clang/lib/Frontend/CompileJobCache.cpp b/clang/lib/Frontend/CompileJobCache.cpp index 505a67c29f99a..549d2c9c15c88 100644 --- a/clang/lib/Frontend/CompileJobCache.cpp +++ b/clang/lib/Frontend/CompileJobCache.cpp @@ -12,7 +12,9 @@ #include "clang/Frontend/CASDependencyCollector.h" #include "clang/Frontend/CompileJobCacheKey.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Lex/PreprocessorOptions.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/CASOutputBackend.h" #include "llvm/RemoteCachingService/Client.h" @@ -79,6 +81,11 @@ class ObjectStoreCachingOutputs : public CompileJobCache::CachingOutputs { CASOutputs = llvm::makeIntrusiveRefCnt(*CAS); } + Expected> + replayCachedResult(const llvm::cas::CASID &ResultCacheKey, + clang::cas::CompileJobCacheResult &Result, + bool JustComputedResult); + private: Expected tryReplayCachedResult(const llvm::cas::CASID &ResultCacheKey) override; @@ -482,6 +489,49 @@ bool CompileJobCache::finishComputedResult(CompilerInstance &Clang, return true; } +Expected> CompileJobCache::replayCachedResult( + std::shared_ptr Invok, const llvm::cas::CASID &CacheKey, + cas::CompileJobCacheResult &CachedResult, SmallVectorImpl &DiagText) { + CompilerInstance Clang; + Clang.setInvocation(std::move(Invok)); + llvm::raw_svector_ostream DiagOS(DiagText); + Clang.createDiagnostics( + new TextDiagnosticPrinter(DiagOS, &Clang.getDiagnosticOpts())); + Clang.setVerboseOutputStream(DiagOS); + + auto FinishDiagnosticClient = + llvm::make_scope_exit([&]() { Clang.getDiagnosticClient().finish(); }); + + llvm::PrefixMapper PrefixMapper; + llvm::SmallVector Split; + llvm::MappedPrefix::transformJoinedIfValid( + Clang.getFrontendOpts().PathPrefixMappings, Split); + for (const auto &MappedPrefix : Split) { + // We use the inverse mapping because the \p PrefixMapper will be used for + // de-canonicalization of paths. + PrefixMapper.add(MappedPrefix.getInverse()); + } + + assert(!Clang.getDiagnostics().hasErrorOccurred()); + + ObjectStoreCachingOutputs CachingOutputs(Clang, std::move(PrefixMapper), + /*CAS*/ nullptr, /*Cache*/ nullptr); + + std::optional Ret; + if (Error E = CachingOutputs + .replayCachedResult(CacheKey, CachedResult, + /*JustComputedResult*/ false) + .moveInto(Ret)) + return E; + + if (Clang.getDiagnostics().hasErrorOccurred()) + return llvm::createStringError(llvm::inconvertibleErrorCode(), + "error diagnostic during replay: " + + DiagOS.str()); + + return Ret; +} + Expected ObjectStoreCachingOutputs::writeOutputs( const llvm::cas::CASID &ResultCacheKey) { DiagnosticsEngine &Diags = Clang.getDiagnostics(); @@ -554,6 +604,22 @@ std::optional ObjectStoreCachingOutputs::replayCachedResult( if (Error E = Schema.load(ResultID).moveInto(Result)) llvm::report_fatal_error(std::move(E)); + std::optional Ret; + // FIXME: Stop calling report_fatal_error(). + if (Error E = replayCachedResult(ResultCacheKey, *Result, JustComputedResult) + .moveInto(Ret)) + llvm::report_fatal_error(std::move(E)); + + return Ret; +} + +Expected> ObjectStoreCachingOutputs::replayCachedResult( + const llvm::cas::CASID &ResultCacheKey, + clang::cas::CompileJobCacheResult &Result, bool JustComputedResult) { + if (JustComputedResult) + return std::nullopt; + + llvm::cas::ObjectStore &CAS = Result.getCAS(); DiagnosticsEngine &Diags = Clang.getDiagnostics(); bool HasMissingOutput = false; std::optional SerialDiags; @@ -563,7 +629,7 @@ std::optional ObjectStoreCachingOutputs::replayCachedResult( if (!Obj.has_value()) { Diags.Report(diag::remark_compile_job_cache_backend_output_not_found) << clang::cas::CompileJobCacheResult::getOutputKindName(O.Kind) - << ResultCacheKey.toString() << CAS->getID(O.Object).toString(); + << ResultCacheKey.toString() << CAS.getID(O.Object).toString(); HasMissingOutput = true; return Error::success(); } @@ -593,7 +659,7 @@ std::optional ObjectStoreCachingOutputs::replayCachedResult( if (O.Kind == OutputKind::Dependencies) { llvm::raw_svector_ostream OS(ContentsStorage); if (auto E = CASDependencyCollector::replay( - Clang.getDependencyOutputOpts(), *CAS, *Obj, OS)) + Clang.getDependencyOutputOpts(), CAS, *Obj, OS)) return E; Contents = ContentsStorage; } else { @@ -608,9 +674,8 @@ std::optional ObjectStoreCachingOutputs::replayCachedResult( return Output->commit(); }; - // FIXME: Stop calling report_fatal_error(). - if (auto Err = Result->forEachLoadedOutput(processOutput)) - llvm::report_fatal_error(std::move(Err)); + if (auto Err = Result.forEachLoadedOutput(processOutput)) + return std::move(Err); if (HasMissingOutput) { Diags.Report(diag::remark_compile_job_cache_miss) @@ -620,12 +685,11 @@ std::optional ObjectStoreCachingOutputs::replayCachedResult( if (!JustComputedResult) { Diags.Report(diag::remark_compile_job_cache_hit) - << ResultCacheKey.toString() << CAS->getID(ResultID).toString(); + << ResultCacheKey.toString() << Result.getID().toString(); if (SerialDiags) { - // FIXME: Stop calling report_fatal_error(). if (Error E = replayCachedDiagnostics(SerialDiags->getData())) - llvm::report_fatal_error(std::move(E)); + return std::move(E); } } diff --git a/clang/lib/Frontend/CompileJobCacheResult.cpp b/clang/lib/Frontend/CompileJobCacheResult.cpp index 73a177d016cfb..7d666485e1962 100644 --- a/clang/lib/Frontend/CompileJobCacheResult.cpp +++ b/clang/lib/Frontend/CompileJobCacheResult.cpp @@ -70,6 +70,10 @@ Error CompileJobCacheResult::forEachLoadedOutput( return Error::success(); } +CompileJobCacheResult::Output CompileJobCacheResult::getOutput(size_t I) const { + return Output{getOutputObject(I), getOutputKind(I)}; +} + Optional CompileJobCacheResult::getOutput(OutputKind Kind) const { size_t Count = getNumOutputs(); diff --git a/clang/test/CAS/libclang-replay-job.c b/clang/test/CAS/libclang-replay-job.c new file mode 100644 index 0000000000000..72638feb0c6e2 --- /dev/null +++ b/clang/test/CAS/libclang-replay-job.c @@ -0,0 +1,61 @@ +// REQUIRES: shell + +// RUN: rm -rf %t && mkdir -p %t +// RUN: llvm-cas --cas %t/cas --ingest --data %s > %t/casid +// +// RUN: %clang -cc1 -triple x86_64-apple-macos11 \ +// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \ +// RUN: -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \ +// RUN: -dependency-file %t/t1.d -MT deps -emit-obj -o %t/output1.o %s +// RUN: %clang -cc1 -triple x86_64-apple-macos11 \ +// RUN: -fcas-path %t/cas -fcas-fs @%t/casid -fcache-compile-job \ +// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \ +// RUN: -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging \ +// RUN: -serialize-diagnostic-file %t/t1.dia -dependency-file %t/t1.d -MT deps \ +// RUN: -Rcompile-job-cache-hit -emit-obj -o %t/output1.o %s 2> %t/output1.txt + +// Verify the warning was recorded and we compare populated .dia files. +// RUN: c-index-test -read-diagnostics %t/t1.dia 2>&1 | FileCheck %s --check-prefix=DIAGS +// DIAGS: warning: some warning + +// RUN: cat %t/output1.txt | grep llvmcas | sed \ +// RUN: -e "s/^.*hit for '//" \ +// RUN: -e "s/' .*$//" > %t/cache-key + +// Delete the "local" cache and use the "upstream" one to re-materialize the outputs locally. +// RUN: rm -rf %t/cas +// RUN: c-index-test core -materialize-cached-job -cas-path %t/cas @%t/cache-key \ +// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \ +// RUN: -fcas-plugin-option upstream-path=%t/cas-upstream -fcas-plugin-option no-logging + +// RUN: c-index-test core -replay-cached-job -cas-path %t/cas @%t/cache-key \ +// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \ +// RUN: -fcas-plugin-option no-logging \ +// RUN: -- -cc1 \ +// RUN: -serialize-diagnostic-file %t/t2.dia -Rcompile-job-cache-hit \ +// RUN: -dependency-file %t/t2.d -MT deps \ +// RUN: -o %t/output2.o 2> %t/output2.txt + +// RUN: diff %t/output1.o %t/output2.o +// RUN: diff -u %t/output1.txt %t/output2.txt +// RUN: diff %t/t1.dia %t/t2.dia +// RUN: diff -u %t/t1.d %t/t2.d + +// Check with `-working-dir` flag. +// RUN: mkdir -p %t/a/b +// RUN: cd %t/a +// RUN: c-index-test core -replay-cached-job -cas-path %t/cas @%t/cache-key \ +// RUN: -fcas-plugin-path %llvmshlibdir/libCASPluginTest%pluginext \ +// RUN: -fcas-plugin-option no-logging \ +// RUN: -working-dir %t/a/b \ +// RUN: -- -cc1 %t/a/b \ +// RUN: -serialize-diagnostic-file rel.dia -Rcompile-job-cache-hit \ +// RUN: -dependency-file rel.d -MT deps \ +// RUN: -o reloutput.o + +// RUN: diff %t/output1.o %t/a/b/reloutput.o +// RUN: diff -u %t/t1.d %t/a/b/rel.d +// FIXME: Get clang's `-working-directory` to affect relative path for serialized diagnostics. + +#warning some warning diff --git a/clang/test/ClangScanDeps/error-c-api.cpp b/clang/test/ClangScanDeps/error-c-api.cpp index d918c3c13ca1e..b40319fcb6313 100644 --- a/clang/test/ClangScanDeps/error-c-api.cpp +++ b/clang/test/ClangScanDeps/error-c-api.cpp @@ -1,4 +1,4 @@ -// RUN: not c-index-test core -scan-deps %S -- clang_tool %s -I %S/Inputs 2>&1 | FileCheck %s +// RUN: not c-index-test core -scan-deps -working-dir %S -- clang_tool %s -I %S/Inputs 2>&1 | FileCheck %s #include "missing.h" diff --git a/clang/test/ClangScanDeps/flags-c-api.cpp b/clang/test/ClangScanDeps/flags-c-api.cpp index 95628395ce150..55bd3c8b36212 100644 --- a/clang/test/ClangScanDeps/flags-c-api.cpp +++ b/clang/test/ClangScanDeps/flags-c-api.cpp @@ -1,4 +1,4 @@ -// RUN: c-index-test core -scan-deps %S -- clang_tool %s -I %S/Inputs | FileCheck %s +// RUN: c-index-test core -scan-deps -working-dir %S -- clang_tool %s -I %S/Inputs | FileCheck %s #include "header.h" diff --git a/clang/test/ClangScanDeps/mllvm-double-option-error-c-api.c b/clang/test/ClangScanDeps/mllvm-double-option-error-c-api.c index 56144108e1326..6e734f73c65ec 100644 --- a/clang/test/ClangScanDeps/mllvm-double-option-error-c-api.c +++ b/clang/test/ClangScanDeps/mllvm-double-option-error-c-api.c @@ -1,4 +1,4 @@ -// RUN: c-index-test core -scan-deps %S -- clang_tool -Dmz -mllvm -asan-instrumentation-with-call-threshold=0 -mllvm -asan-instrumentation-with-call-threshold=0 %s -I %S/Inputs | FileCheck %s +// RUN: c-index-test core -scan-deps -working-dir %S -- clang_tool -Dmz -mllvm -asan-instrumentation-with-call-threshold=0 -mllvm -asan-instrumentation-with-call-threshold=0 %s -I %S/Inputs | FileCheck %s #ifdef mz #include "header.h" diff --git a/clang/test/ClangScanDeps/modules-order-c-api.c b/clang/test/ClangScanDeps/modules-order-c-api.c index c16e0a4609fc5..a43dcfe1a20cc 100644 --- a/clang/test/ClangScanDeps/modules-order-c-api.c +++ b/clang/test/ClangScanDeps/modules-order-c-api.c @@ -2,9 +2,9 @@ // RUN: split-file %s %t // Scan repeatedly -// RUN: c-index-test core -scan-deps %S -- clang_tool -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache -fimplicit-modules -fimplicit-module-maps 2>&1 > %t/output1 -// RUN: c-index-test core -scan-deps %S -- clang_tool -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache -fimplicit-modules -fimplicit-module-maps 2>&1 > %t/output2 -// RUN: c-index-test core -scan-deps %S -- clang_tool -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache -fimplicit-modules -fimplicit-module-maps 2>&1 > %t/output3 +// RUN: c-index-test core -scan-deps -working-dir %S -- clang_tool -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache -fimplicit-modules -fimplicit-module-maps 2>&1 > %t/output1 +// RUN: c-index-test core -scan-deps -working-dir %S -- clang_tool -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache -fimplicit-modules -fimplicit-module-maps 2>&1 > %t/output2 +// RUN: c-index-test core -scan-deps -working-dir %S -- clang_tool -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache -fimplicit-modules -fimplicit-module-maps 2>&1 > %t/output3 // Ensure the output is identical each time // RUN: diff %t/output1 %t/output2 diff --git a/clang/test/ClangScanDeps/modules-outputs-c-api.c b/clang/test/ClangScanDeps/modules-outputs-c-api.c index cebdbdd32f9e0..4b82423f2ebc5 100644 --- a/clang/test/ClangScanDeps/modules-outputs-c-api.c +++ b/clang/test/ClangScanDeps/modules-outputs-c-api.c @@ -1,7 +1,7 @@ // RUN: rm -rf %t // RUN: split-file %s %t -// RUN: c-index-test core -scan-deps %t -output-dir %t/out -- \ +// RUN: c-index-test core -scan-deps -working-dir %t -output-dir %t/out -- \ // RUN: clang_tool -c %t/tu.c -fmodules -fmodules-cache-path=%t/cache \ // RUN: -fimplicit-modules -fimplicit-module-maps \ // RUN: -serialize-diagnostics %t/tu.diag -MD -MF %t/tu.d -o %t/tu.o \ @@ -14,7 +14,7 @@ // NONE: build-args: // NONE-SAME: -fmodule-file={{(Mod=)?}}[[PREFIX]]/out/Mod_{{.*}}.pcm -// RUN: c-index-test core -scan-deps %t -output-dir %t/out -serialize-diagnostics -- \ +// RUN: c-index-test core -scan-deps -working-dir %t -output-dir %t/out -serialize-diagnostics -- \ // RUN: clang_tool -c %t/tu.c -fmodules -fmodules-cache-path=%t/cache \ // RUN: -fimplicit-modules -fimplicit-module-maps \ // RUN: -serialize-diagnostics %t/tu.diag -MD -MF %t/tu.d -o %t/tu.o \ @@ -29,7 +29,7 @@ // DIAGS: build-args: // DIAGS-SAME: -fmodule-file={{(Mod=)?}}[[PREFIX]]/out/Mod_{{.*}}.pcm -// RUN: c-index-test core -scan-deps %t -output-dir %t/out -dependency-file -- \ +// RUN: c-index-test core -scan-deps -working-dir %t -output-dir %t/out -dependency-file -- \ // RUN: clang_tool -c %t/tu.c -fmodules -fmodules-cache-path=%t/cache \ // RUN: -fimplicit-modules -fimplicit-module-maps \ // RUN: -serialize-diagnostics %t/tu.diag -MD -MF %t/tu.d -o %t/tu.o \ @@ -44,7 +44,7 @@ // DEPS: build-args: // DEPS-SAME: -fmodule-file={{(Mod=)?}}[[PREFIX]]/out/Mod_{{.*}}.pcm -// RUN: c-index-test core -scan-deps %t -output-dir %t/out -dependency-file -dependency-target foo -- \ +// RUN: c-index-test core -scan-deps -working-dir %t -output-dir %t/out -dependency-file -dependency-target foo -- \ // RUN: clang_tool -c %t/tu.c -fmodules -fmodules-cache-path=%t/cache \ // RUN: -fimplicit-modules -fimplicit-module-maps \ // RUN: -serialize-diagnostics %t/tu.diag -MD -MF %t/tu.d -o %t/tu.o \ @@ -57,7 +57,7 @@ // DEPS_MT1: build-args: // DEPS_MT1-SAME: -fmodule-file={{(Mod=)?}}[[PREFIX]]/out/Mod_{{.*}}.pcm -// RUN: c-index-test core -scan-deps %t -output-dir %t/out -dependency-file -dependency-target foo -dependency-target bar -- \ +// RUN: c-index-test core -scan-deps -working-dir %t -output-dir %t/out -dependency-file -dependency-target foo -dependency-target bar -- \ // RUN: clang_tool -c %t/tu.c -fmodules -fmodules-cache-path=%t/cache \ // RUN: -fimplicit-modules -fimplicit-module-maps \ // RUN: -serialize-diagnostics %t/tu.diag -MD -MF %t/tu.d -o %t/tu.o \ @@ -73,7 +73,7 @@ // RUN: echo 'this_target_name_is_longer_than_the_256_byte_initial_buffer_size_to_test_that_we_alloc_and_call_again_with_a_sufficient_buffer_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX_end' > %t/target-name.txt // RUN: cat %t/target-name.txt > %t/long.txt -// RUN: c-index-test core -scan-deps %t -output-dir %t/out -dependency-file \ +// RUN: c-index-test core -scan-deps -working-dir %t -output-dir %t/out -dependency-file \ // RUN: -dependency-target @%t/target-name.txt -- \ // RUN: clang_tool -c %t/tu.c -fmodules -fmodules-cache-path=%t/cache \ // RUN: -fimplicit-modules -fimplicit-module-maps \ diff --git a/clang/test/ClangScanDeps/simple-c-api.cpp b/clang/test/ClangScanDeps/simple-c-api.cpp index 93de517631454..bfc92ff32a136 100644 --- a/clang/test/ClangScanDeps/simple-c-api.cpp +++ b/clang/test/ClangScanDeps/simple-c-api.cpp @@ -1,4 +1,4 @@ -// RUN: c-index-test core -scan-deps %S -- clang_tool %s -I %S/Inputs | FileCheck %s +// RUN: c-index-test core -scan-deps -working-dir %S -- clang_tool %s -I %S/Inputs | FileCheck %s #include "header.h" diff --git a/clang/test/Index/Core/scan-deps-by-mod-name.m b/clang/test/Index/Core/scan-deps-by-mod-name.m index 556ffa1e48fe3..af3c84c5cb316 100644 --- a/clang/test/Index/Core/scan-deps-by-mod-name.m +++ b/clang/test/Index/Core/scan-deps-by-mod-name.m @@ -3,7 +3,7 @@ // RUN: echo %S > %t.result // RUN: echo %S > %t_v2.result // -// RUN: c-index-test core --scan-deps-by-mod-name -output-dir %t -module-name=ModA %S -- %clang -c -I %S/Inputs/module \ +// RUN: c-index-test core --scan-deps-by-mod-name -output-dir %t -module-name=ModA -working-dir %S -- %clang -c -I %S/Inputs/module \ // RUN: -fmodules -fmodules-cache-path=%t.mcp \ // RUN: -o FoE.o -x objective-c >> %t.result // RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DOUTPUTS=%/t diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index afeba129d9ecc..33c369a2d528e 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -1,24 +1,24 @@ // RUN: rm -rf %t -// RUN: c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ +// RUN: c-index-test core --scan-deps -working-dir %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ // RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.result // RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE -// RUN: env CLANG_CACHE_USE_CASFS_DEPSCAN=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ +// RUN: env CLANG_CACHE_USE_CASFS_DEPSCAN=1 c-index-test core --scan-deps -working-dir %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ // RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s > %t.casfs.result // RUN: cat %t.casfs.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -// RUN: env CLANG_CACHE_USE_INCLUDE_TREE=1 c-index-test core --scan-deps %S -output-dir=%t -cas-path %t/cas \ +// RUN: env CLANG_CACHE_USE_INCLUDE_TREE=1 c-index-test core --scan-deps -working-dir %S -output-dir=%t -cas-path %t/cas \ // RUN: -- %clang -c -I %S/Inputs/module \ // RUN: -fmodules -fmodules-cache-path=%t/mcpit \ // RUN: -o FoE.o -x objective-c %s > %t.includetree.result // RUN: cat %t.includetree.result | sed 's/\\/\//g' | FileCheck %s -DPREFIX=%S -DOUTPUTS=%/t -check-prefix=INCLUDE_TREE -// RUN: c-index-test core --scan-deps %S -output-dir=%t \ +// RUN: c-index-test core --scan-deps -working-dir %S -output-dir=%t \ // RUN: -- %clang -c -I %S/Inputs/module \ // RUN: -fmodules -fmodules-cache-path=%t/mcp \ // RUN: -o FoE.o -x objective-c %s | FileCheck %s -check-prefix=NO_CAS diff --git a/clang/test/Index/Core/scan-deps-with-diags.m b/clang/test/Index/Core/scan-deps-with-diags.m index a47d3646385ff..680ecbe48967a 100644 --- a/clang/test/Index/Core/scan-deps-with-diags.m +++ b/clang/test/Index/Core/scan-deps-with-diags.m @@ -1,4 +1,4 @@ -// RUN: not c-index-test core --scan-deps %S -output-dir=%t -- \ +// RUN: not c-index-test core --scan-deps -working-dir %S -output-dir=%t -- \ // RUN: %clang -c %s -o %t/t.o 2> %t.err.txt // RUN: FileCheck -input-file=%t.err.txt %s diff --git a/clang/test/Index/Core/scan-deps.m b/clang/test/Index/Core/scan-deps.m index bd33b0fd3925e..44adc11c4ec24 100644 --- a/clang/test/Index/Core/scan-deps.m +++ b/clang/test/Index/Core/scan-deps.m @@ -4,12 +4,12 @@ // RUN: echo %S > %t_savetemps.result // RUN: echo %S > %t_v3.result // -// RUN: c-index-test core --scan-deps %S -output-dir=%t -- %clang -c -I %S/Inputs/module \ +// RUN: c-index-test core --scan-deps -working-dir %S -output-dir=%t -- %clang -c -I %S/Inputs/module \ // RUN: -fmodules -fmodules-cache-path=%t.mcp \ // RUN: -o FoE.o -x objective-c %s >> %t.result // RUN: cat %t.result | sed 's/\\/\//g' | FileCheck %s -DOUTPUTS=%/t --check-prefixes=CHECK,CC1 -// RUN: c-index-test core --scan-deps %S -output-dir=%t -- %clang -c -I %S/Inputs/module \ +// RUN: c-index-test core --scan-deps -working-dir %S -output-dir=%t -- %clang -c -I %S/Inputs/module \ // RUN: -fmodules -fmodules-cache-path=%t.mcp -save-temps=obj \ // RUN: -o FoE.o -x objective-c %s >> %t_savetemps.result // RUN: cat %t_savetemps.result | sed 's/\\/\//g' | FileCheck %s -DOUTPUTS=%/t --check-prefixes=CHECK,SAVETEMPS diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index d9aa02fce3228..d2072e169cbe4 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -37,6 +37,7 @@ #include "llvm/Support/Signals.h" #include "llvm/Support/StringSaver.h" #include "llvm/Support/raw_ostream.h" +#include #include using namespace clang; @@ -57,6 +58,8 @@ enum class ActionType { AggregateAsJSON, ScanDeps, ScanDepsByModuleName, + MaterializeCachedJob, + ReplayCachedJob, WatchDir, }; @@ -81,6 +84,10 @@ Action(cl::desc("Action:"), cl::init(ActionType::None), "Get file dependencies"), clEnumValN(ActionType::ScanDepsByModuleName, "scan-deps-by-mod-name", "Get file dependencies by module name alone"), + clEnumValN(ActionType::MaterializeCachedJob, "materialize-cached-job", + "Materialize cached compilation data from upstream CAS"), + clEnumValN(ActionType::ReplayCachedJob, "replay-cached-job", + "Replay a cached compilation from the CAS"), clEnumValN(ActionType::WatchDir, "watch-dir", "Watch directory for file events")), cl::cat(IndexTestCoreCategory)); @@ -139,6 +146,12 @@ static cl::list DependencyTargets( cl::desc("module builds should use the given dependency target(s)")); static llvm::cl::opt CASPath("cas-path", llvm::cl::desc("Path for on-disk CAS/cache.")); +static llvm::cl::opt + CASPluginPath("fcas-plugin-path", llvm::cl::desc("Path for CAS plugin")); +static cl::list CASPluginOpts("fcas-plugin-option", + cl::desc("Plugin CAS Options")); +static llvm::cl::opt + WorkingDir("working-dir", llvm::cl::desc("Path for working directory")); } } // anonymous namespace @@ -680,7 +693,7 @@ static void printSymbolNameAndUSR(const clang::Module *Mod, raw_ostream &OS) { static int scanDeps(ArrayRef Args, std::string WorkingDirectory, bool SerializeDiags, bool DependencyFile, ArrayRef DepTargets, std::string OutputPath, - Optional CASPath, + CXCASDatabases DBs, Optional ModuleName = std::nullopt) { CXDependencyScannerServiceOptions Opts = clang_experimental_DependencyScannerServiceOptions_create(); @@ -690,25 +703,9 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DependencyScannerServiceOptions_setDependencyMode( Opts, CXDependencyMode_Full); - CXString Error; - if (CASPath) { - CXCASOptions CASOpts = clang_experimental_cas_Options_create(); - auto CleanupCASOpts = llvm::make_scope_exit( - [&] { clang_experimental_cas_Options_dispose(CASOpts); }); - clang_experimental_cas_Options_setOnDiskPath(CASOpts, CASPath->c_str()); - CXCASDatabases DBs = - clang_experimental_cas_Databases_create(CASOpts, &Error); - auto CleanupCaches = llvm::make_scope_exit( - [&] { clang_experimental_cas_Databases_dispose(DBs); }); - if (!DBs) { - llvm::errs() << "error: failed to create cache instances\n"; - llvm::errs() << clang_getCString(Error) << "\n"; - clang_disposeString(Error); - return 1; - } + if (DBs) clang_experimental_DependencyScannerServiceOptions_setCASDatabases(Opts, DBs); - } CXDependencyScannerService Service = clang_experimental_DependencyScannerService_create_v1(Opts); @@ -864,6 +861,133 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, return 1; } +static int materializeCachedJob(std::string CacheKey, CXCASDatabases DBs) { + struct CompResult { + CXCASCachedCompilation Comp = nullptr; + CXError Err = nullptr; + }; + std::promise CompPromise; + auto CompFuture = CompPromise.get_future(); + struct CompCall { + std::promise Promise; + }; + CompCall *CallCtx = new CompCall{std::move(CompPromise)}; + clang_experimental_cas_getCachedCompilation_async( + DBs, CacheKey.c_str(), /*Globally*/ true, CallCtx, + [](void *Ctx, CXCASCachedCompilation Comp, CXError Err) { + std::unique_ptr CallCtx(static_cast(Ctx)); + CallCtx->Promise.set_value(CompResult{Comp, Err}); + }, + /*cancelToken*/ nullptr); + CompResult Res = CompFuture.get(); + CXCASCachedCompilation CComp = Res.Comp; + + auto CleanupCachedComp = llvm::make_scope_exit([&] { + if (CComp) + clang_experimental_cas_CachedCompilation_dispose(CComp); + if (Res.Err) + clang_Error_dispose(Res.Err); + }); + if (!CComp) { + if (Res.Err) { + llvm::errs() << clang_Error_getDescription(Res.Err) << "\n"; + } else { + llvm::errs() << "cache key was not found\n"; + } + return 1; + } + + for (unsigned + I = 0, + E = clang_experimental_cas_CachedCompilation_getNumOutputs(CComp); + I != E; ++I) { + if (clang_experimental_cas_CachedCompilation_isOutputMaterialized(CComp, I)) + continue; + CXString OutputID = + clang_experimental_cas_CachedCompilation_getOutputCASIDString(CComp, I); + auto CleanupOutputID = + llvm::make_scope_exit([&] { clang_disposeString(OutputID); }); + + struct LoadResult { + CXCASObject Obj = nullptr; + CXError Err = nullptr; + }; + std::promise LoadPromise; + auto LoadFuture = LoadPromise.get_future(); + struct LoadCall { + std::promise Promise; + }; + LoadCall *CallCtx = new LoadCall{std::move(LoadPromise)}; + clang_experimental_cas_loadObjectByString_async( + DBs, clang_getCString(OutputID), CallCtx, + [](void *Ctx, CXCASObject Obj, CXError Err) { + std::unique_ptr CallCtx(static_cast(Ctx)); + CallCtx->Promise.set_value(LoadResult{Obj, Err}); + }, + /*cancelToken*/ nullptr); + + LoadResult Res = LoadFuture.get(); + CXCASObject CASObj = Res.Obj; + + auto CleanupLoadObj = llvm::make_scope_exit([&] { + if (CASObj) + clang_experimental_cas_CASObject_dispose(CASObj); + if (Res.Err) + clang_Error_dispose(Res.Err); + }); + + if (!CASObj) { + if (Res.Err) { + llvm::errs() << clang_Error_getDescription(Res.Err) << "\n"; + } else { + llvm::errs() << "compilation output ID was not found\n"; + } + return 1; + } + if (!clang_experimental_cas_CachedCompilation_isOutputMaterialized(CComp, + I)) + report_fatal_error("output was not materialized?"); + } + + return 0; +} + +static int replayCachedJob(ArrayRef Args, + std::string WorkingDirectory, std::string CacheKey, + CXCASDatabases DBs) { + CXError Err = nullptr; + CXCASCachedCompilation CComp = clang_experimental_cas_getCachedCompilation( + DBs, CacheKey.c_str(), /*Globally*/ false, &Err); + auto CleanupCachedComp = llvm::make_scope_exit( + [&] { clang_experimental_cas_CachedCompilation_dispose(CComp); }); + if (!CComp) { + if (Err) { + llvm::errs() << clang_Error_getDescription(Err) << "\n"; + clang_Error_dispose(Err); + } else { + llvm::errs() << "cache key was not found\n"; + } + return 1; + } + + CXCASReplayResult ReplayRes = clang_experimental_cas_replayCompilation( + CComp, Args.size(), Args.data(), + WorkingDirectory.empty() ? nullptr : WorkingDirectory.c_str(), + /*reserved*/ nullptr, &Err); + auto CleanupReplayRes = llvm::make_scope_exit( + [&] { clang_experimental_cas_ReplayResult_dispose(ReplayRes); }); + if (!ReplayRes) { + llvm::errs() << clang_Error_getDescription(Err) << "\n"; + clang_Error_dispose(Err); + return 1; + } + + CXString DiagText = clang_experimental_cas_ReplayResult_getStderr(ReplayRes); + llvm::errs() << clang_getCString(DiagText); + clang_disposeString(DiagText); + return 0; +} + static void printSymbol(const IndexRecordDecl &Rec, raw_ostream &OS) { printSymbolInfo(Rec.SymInfo, OS); OS << " | "; @@ -1169,29 +1293,88 @@ int indextest_core_main(int argc, const char **argv) { ? std::nullopt : Optional(options::CASPath); + CXCASOptions CASOpts = nullptr; + CXCASDatabases DBs = nullptr; + auto CleanupCASOpts = llvm::make_scope_exit([&] { + if (CASOpts) + clang_experimental_cas_Options_dispose(CASOpts); + }); + auto CleanupCaches = llvm::make_scope_exit([&] { + if (DBs) + clang_experimental_cas_Databases_dispose(DBs); + }); + + if (CASPath) { + CASOpts = clang_experimental_cas_Options_create(); + clang_experimental_cas_Options_setOnDiskPath(CASOpts, CASPath->c_str()); + if (!options::CASPluginPath.empty()) + clang_experimental_cas_Options_setPluginPath( + CASOpts, options::CASPluginPath.c_str()); + for (const auto &PluginOpt : options::CASPluginOpts) { + auto [Name, Val] = StringRef(PluginOpt).split('='); + std::string NameStr(Name); + std::string ValStr(Val); + clang_experimental_cas_Options_setPluginOption(CASOpts, NameStr.c_str(), + ValStr.c_str()); + } + CXString Error; + DBs = clang_experimental_cas_Databases_create(CASOpts, &Error); + if (!DBs) { + llvm::errs() << "error: failed to create cas/cache databases\n"; + llvm::errs() << clang_getCString(Error) << "\n"; + clang_disposeString(Error); + return 1; + } + } + if (options::Action == ActionType::ScanDeps) { - if (options::InputFiles.empty()) { - errs() << "error: missing working directory\n"; + if (options::WorkingDir.empty()) { + errs() << "error: missing -working-dir\n"; return 1; } - return scanDeps(CompArgs, options::InputFiles[0], options::SerializeDiags, + return scanDeps(CompArgs, options::WorkingDir, options::SerializeDiags, options::DependencyFile, options::DependencyTargets, - options::OutputDir, CASPath); + options::OutputDir, DBs); } if (options::Action == ActionType::ScanDepsByModuleName) { // InputFiles should be set to the working directory name. - if (options::InputFiles.empty()) { - errs() << "error: missing working directory\n"; + if (options::WorkingDir.empty()) { + errs() << "error: missing -working-dir\n"; return 1; } if (options::ModuleName.empty()) { errs() << "error: missing module name\n"; return 1; } - return scanDeps(CompArgs, options::InputFiles[0], options::SerializeDiags, + return scanDeps(CompArgs, options::WorkingDir, options::SerializeDiags, options::DependencyFile, options::DependencyTargets, - options::OutputDir, CASPath, options::ModuleName); + options::OutputDir, DBs, options::ModuleName); + } + + if (options::Action == ActionType::MaterializeCachedJob) { + if (options::InputFiles.empty()) { + errs() << "error: missing cache key\n"; + return 1; + } + if (!DBs) { + errs() << "error: CAS was not configured\n"; + return 1; + } + return materializeCachedJob(options::InputFiles[0], DBs); + } + + if (options::Action == ActionType::ReplayCachedJob) { + if (options::InputFiles.empty()) { + errs() << "error: missing cache key\n"; + return 1; + } + if (!DBs) { + errs() << "error: CAS was not configured\n"; + return 1; + } + return replayCachedJob(CompArgs, options::WorkingDir, + options::InputFiles[0], DBs); } if (options::Action == ActionType::WatchDir) { diff --git a/clang/tools/driver/features.json b/clang/tools/driver/features.json index 3989988e1bff0..3d0d1ad873aef 100644 --- a/clang/tools/driver/features.json +++ b/clang/tools/driver/features.json @@ -18,6 +18,9 @@ { "name": "depscan-prefix-map" }, + { + "name": "libclang-cache-queries" + }, { "name": "deployment-target-environment-variables", "value": [ diff --git a/clang/tools/libclang/CCAS.cpp b/clang/tools/libclang/CCAS.cpp index 5e15ffda0535d..68650ba0215c9 100644 --- a/clang/tools/libclang/CCAS.cpp +++ b/clang/tools/libclang/CCAS.cpp @@ -9,17 +9,81 @@ #include "clang-c/CAS.h" #include "CASUtils.h" +#include "CXError.h" #include "CXString.h" #include "clang/Basic/LLVM.h" #include "clang/CAS/CASOptions.h" +#include "clang/Frontend/CompileJobCache.h" +#include "clang/Frontend/CompilerInvocation.h" #include "clang/Frontend/TextDiagnosticPrinter.h" +#include "llvm/ADT/ScopeExit.h" #include "llvm/CAS/ActionCache.h" #include "llvm/CAS/ObjectStore.h" #include "llvm/Support/Path.h" using namespace clang; using namespace clang::cas; +using llvm::Error; + +namespace { + +struct WrappedCASObject { + ObjectProxy Obj; + std::shared_ptr CAS; +}; + +struct WrappedCachedCompilation { + CASID CacheKey; + clang::cas::CompileJobCacheResult CachedResult; + std::shared_ptr CAS; + std::shared_ptr AC; + + static CXCASCachedCompilation + fromResultID(Expected> ResultID, CASID CacheKey, + const std::shared_ptr &CAS, + const std::shared_ptr &AC, + CXError *OutError); +}; + +struct WrappedReplayResult { + SmallString<256> DiagText; +}; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(WrappedCASObject, CXCASObject) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(WrappedCachedCompilation, + CXCASCachedCompilation) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(WrappedReplayResult, CXCASReplayResult) + +} // anonymous namespace + +CXCASCachedCompilation WrappedCachedCompilation::fromResultID( + Expected> ResultID, CASID CacheKey, + const std::shared_ptr &CAS, + const std::shared_ptr &AC, CXError *OutError) { + + auto failure = [OutError](Error &&E) -> CXCASCachedCompilation { + if (OutError) + *OutError = cxerror::create(std::move(E)); + return nullptr; + }; + + if (!ResultID) + return failure(ResultID.takeError()); + if (!*ResultID) + return nullptr; + + auto OptResultRef = CAS->getReference(**ResultID); + if (!OptResultRef) + return nullptr; + + clang::cas::CompileJobResultSchema Schema(*CAS); + auto CachedResult = Schema.load(*OptResultRef); + if (!CachedResult) + return failure(CachedResult.takeError()); + return wrap(new WrappedCachedCompilation{std::move(CacheKey), + std::move(*CachedResult), CAS, AC}); +} CXCASOptions clang_experimental_cas_Options_create(void) { return wrap(new CASOptions()); @@ -74,6 +138,338 @@ void clang_experimental_cas_Databases_dispose(CXCASDatabases CDBs) { delete unwrap(CDBs); } +CXCASObject clang_experimental_cas_loadObjectByString(CXCASDatabases CDBs, + const char *PrintedID, + CXError *OutError) { + WrappedCASDatabases &DBs = *unwrap(CDBs); + ObjectStore &CAS = *DBs.CAS; + + if (OutError) + *OutError = nullptr; + + auto failure = [OutError](Error &&E) -> CXCASObject { + if (OutError) + *OutError = cxerror::create(std::move(E)); + return nullptr; + }; + + Expected Digest = CAS.parseID(PrintedID); + if (!Digest) + return failure(Digest.takeError()); + Optional Ref = CAS.getReference(*Digest); + if (!Ref) + return nullptr; + + // Visit the graph of the object to ensure it's fully materialized. + + SmallVector ObjectsToLoad; + ObjectsToLoad.push_back(*Ref); + llvm::SmallDenseSet ObjectsSeen; + + while (!ObjectsToLoad.empty()) { + ObjectRef Ref = ObjectsToLoad.pop_back_val(); + bool Inserted = ObjectsSeen.insert(Ref).second; + if (!Inserted) + continue; + std::optional Obj; + if (Error E = CAS.getProxy(Ref).moveInto(Obj)) + return failure(std::move(E)); + if (Error E = Obj->forEachReference([&ObjectsToLoad](ObjectRef R) -> Error { + ObjectsToLoad.push_back(R); + return Error::success(); + })) + return failure(std::move(E)); + } + + std::optional Obj; + if (Error E = CAS.getProxy(*Ref).moveInto(Obj)) + return failure(std::move(E)); + + if (!Obj) + return nullptr; + return wrap(new WrappedCASObject{std::move(*Obj), DBs.CAS}); +} + +void clang_experimental_cas_loadObjectByString_async( + CXCASDatabases CDBs, const char *PrintedID, void *Ctx, + void (*Callback)(void *Ctx, CXCASObject, CXError), + CXCASCancellationToken *OutToken) { + if (OutToken) + *OutToken = nullptr; + WrappedCASDatabases &DBs = *unwrap(CDBs); + ObjectStore &CAS = *DBs.CAS; + + Expected Digest = CAS.parseID(PrintedID); + if (!Digest) + return Callback(Ctx, nullptr, cxerror::create(Digest.takeError())); + Optional Ref = CAS.getReference(*Digest); + if (!Ref) + return Callback(Ctx, nullptr, nullptr); + + /// Asynchronously visits the graph of the object node to ensure it's fully + /// materialized. + class AsyncObjectLoader + : public std::enable_shared_from_this { + void *Ctx; + void (*Callback)(void *Ctx, CXCASObject, CXError); + std::shared_ptr CAS; + + llvm::SmallDenseSet ObjectsSeen; + unsigned NumPending = 0; + std::optional RootObj; + std::atomic MissingNode{false}; + /// The first error that occurred. + std::optional ErrOccurred; + std::mutex Mutex; + + public: + AsyncObjectLoader(void *Ctx, + void (*Callback)(void *Ctx, CXCASObject, CXError), + std::shared_ptr CAS) + : Ctx(Ctx), Callback(Callback), CAS(std::move(CAS)) {} + + void visit(ObjectRef Ref, bool IsRootNode) { + bool Inserted; + { + std::lock_guard Guard(Mutex); + Inserted = ObjectsSeen.insert(Ref).second; + if (Inserted) + ++NumPending; + } + if (!Inserted) { + finishedNode(); + return; + } + auto This = shared_from_this(); + CAS->getProxyAsync( + Ref, [This, IsRootNode](Expected> Obj) { + auto _1 = llvm::make_scope_exit([&]() { This->finishedNode(); }); + if (!Obj) { + This->encounteredError(Obj.takeError()); + return; + } + if (!*Obj) { + This->MissingNode = true; + return; + } + if (IsRootNode) + This->RootObj = *Obj; + cantFail((*Obj)->forEachReference([&This](ObjectRef Sub) -> Error { + This->visit(Sub, /*IsRootNode*/ false); + return Error::success(); + })); + }); + } + + void finishedNode() { + bool FinishedPending; + { + std::lock_guard Guard(Mutex); + assert(NumPending); + --NumPending; + FinishedPending = (NumPending == 0); + } + if (!FinishedPending) + return; + + if (ErrOccurred) + return Callback(Ctx, nullptr, cxerror::create(std::move(*ErrOccurred))); + if (MissingNode) + return Callback(Ctx, nullptr, nullptr); + return Callback( + Ctx, wrap(new WrappedCASObject{std::move(*RootObj), std::move(CAS)}), + nullptr); + } + + /// Only keeps the first error that occurred. + void encounteredError(Error &&E) { + std::lock_guard Guard(Mutex); + if (ErrOccurred) { + llvm::consumeError(std::move(E)); + return; + } + ErrOccurred = std::move(E); + } + }; + + auto WL = std::make_shared(Ctx, Callback, DBs.CAS); + WL->visit(*Ref, /*IsRootNode*/ true); +} + +void clang_experimental_cas_CASObject_dispose(CXCASObject CObj) { + delete unwrap(CObj); +} + +CXCASCachedCompilation +clang_experimental_cas_getCachedCompilation(CXCASDatabases CDBs, + const char *CacheKey, bool Globally, + CXError *OutError) { + WrappedCASDatabases &DBs = *unwrap(CDBs); + + if (OutError) + *OutError = nullptr; + + auto failure = [OutError](Error &&E) -> CXCASCachedCompilation { + if (OutError) + *OutError = cxerror::create(std::move(E)); + return nullptr; + }; + + Expected KeyID = DBs.CAS->parseID(CacheKey); + if (!KeyID) + return failure(KeyID.takeError()); + + Optional ValID; + if (Error E = DBs.Cache->get(*KeyID, Globally).moveInto(ValID)) + return WrappedCachedCompilation::fromResultID( + std::move(E), *KeyID, DBs.CAS, DBs.Cache, OutError); + return WrappedCachedCompilation::fromResultID( + ValID ? std::optional(*ValID) : std::optional(std::nullopt), *KeyID, DBs.CAS, DBs.Cache, OutError); +} + +void clang_experimental_cas_getCachedCompilation_async( + CXCASDatabases CDBs, const char *CacheKey, bool Globally, void *Ctx, + void (*Callback)(void *Ctx, CXCASCachedCompilation, CXError), + CXCASCancellationToken *OutToken) { + if (OutToken) + *OutToken = nullptr; + WrappedCASDatabases &DBs = *unwrap(CDBs); + + Expected KeyID = DBs.CAS->parseID(CacheKey); + if (!KeyID) + return Callback(Ctx, nullptr, cxerror::create(KeyID.takeError())); + + DBs.Cache->getAsync(*KeyID, Globally, + [KeyID = *KeyID, CAS = DBs.CAS, AC = DBs.Cache, Ctx, + Callback](Expected> ResultID) { + CXError Err = nullptr; + CXCASCachedCompilation CComp = + WrappedCachedCompilation::fromResultID( + std::move(ResultID), std::move(KeyID), + std::move(CAS), std::move(AC), &Err); + Callback(Ctx, CComp, Err); + }); +} + +void clang_experimental_cas_CachedCompilation_dispose( + CXCASCachedCompilation CComp) { + delete unwrap(CComp); +} + +size_t clang_experimental_cas_CachedCompilation_getNumOutputs( + CXCASCachedCompilation CComp) { + return unwrap(CComp)->CachedResult.getNumOutputs(); +} + +CXString clang_experimental_cas_CachedCompilation_getOutputName( + CXCASCachedCompilation CComp, size_t OutputIdx) { + CompileJobCacheResult::Output Output = + unwrap(CComp)->CachedResult.getOutput(OutputIdx); + return cxstring::createRef( + CompileJobCacheResult::getOutputKindName(Output.Kind)); +} + +CXString clang_experimental_cas_CachedCompilation_getOutputCASIDString( + CXCASCachedCompilation CComp, size_t OutputIdx) { + WrappedCachedCompilation &WComp = *unwrap(CComp); + CompileJobCacheResult::Output Output = + WComp.CachedResult.getOutput(OutputIdx); + return cxstring::createDup(WComp.CAS->getID(Output.Object).toString()); +} + +bool clang_experimental_cas_CachedCompilation_isOutputMaterialized( + CXCASCachedCompilation CComp, size_t OutputIdx) { + WrappedCachedCompilation &WComp = *unwrap(CComp); + CompileJobCacheResult::Output Output = + WComp.CachedResult.getOutput(OutputIdx); + bool IsMaterialized = false; + // FIXME: Propagate error to caller instead of calling `report_fatal_error`. + // In practice this should not fail because it checks the local CAS only. + if (Error E = + WComp.CAS->isMaterialized(Output.Object).moveInto(IsMaterialized)) + llvm::report_fatal_error(std::move(E)); + return IsMaterialized; +} + +void clang_experimental_cas_CachedCompilation_makeGlobal( + CXCASCachedCompilation CComp, void *Ctx, + void (*Callback)(void *Ctx, CXError), CXCASCancellationToken *OutToken) { + if (OutToken) + *OutToken = nullptr; + WrappedCachedCompilation &WComp = *unwrap(CComp); + CompileJobCacheResult &CacheResult = WComp.CachedResult; + WComp.AC->putAsync(WComp.CacheKey, CacheResult.getID(), /*Globally=*/true, + [Ctx, Callback](Error E) { + Callback(Ctx, cxerror::create(std::move(E))); + }); +} + +CXCASReplayResult clang_experimental_cas_replayCompilation( + CXCASCachedCompilation CComp, int argc, const char *const *argv, + const char *WorkingDirectory, void * /*reserved*/, CXError *OutError) { + WrappedCachedCompilation &WComp = *unwrap(CComp); + + if (OutError) + *OutError = nullptr; + + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + SmallString<128> DiagsBuffer; + llvm::raw_svector_ostream DiagOS(DiagsBuffer); + auto *DiagsPrinter = new TextDiagnosticPrinter(DiagOS, DiagOpts.get()); + DiagnosticsEngine Diags(DiagID, DiagOpts.get(), DiagsPrinter); + + auto Invok = std::make_shared(); + ArrayRef Argv(argv, argc); + bool Success = CompilerInvocation::CreateFromArgs(*Invok, Argv.drop_front(), + Diags, Argv.front()); + if (!Success) { + if (OutError) + *OutError = cxerror::create(DiagOS.str()); + return nullptr; + } + + StringRef WorkingDir = WorkingDirectory; + if (!WorkingDir.empty()) + Invok->getFileSystemOpts().WorkingDir = WorkingDir; + + SmallString<256> DiagText; + std::optional Ret; + if (Error E = + CompileJobCache::replayCachedResult(std::move(Invok), WComp.CacheKey, + WComp.CachedResult, DiagText) + .moveInto(Ret)) { + if (OutError) + *OutError = cxerror::create(std::move(E)); + else + llvm::consumeError(std::move(E)); + return nullptr; + } + + if (!Ret) + return nullptr; + // If there was no CAS error and the compilation was cached it will be + // 'success', we don't cache compilation failures. + assert(*Ret == 0); + return wrap(new WrappedReplayResult{std::move(DiagText)}); +} + +void clang_experimental_cas_ReplayResult_dispose(CXCASReplayResult CRR) { + delete unwrap(CRR); +} + +CXString clang_experimental_cas_ReplayResult_getStderr(CXCASReplayResult CRR) { + return cxstring::createDup(unwrap(CRR)->DiagText); +} + +void clang_experimental_cas_CancellationToken_cancel(CXCASCancellationToken) { + // FIXME: Implement. +} + +void clang_experimental_cas_CancellationToken_dispose(CXCASCancellationToken) { + // FIXME: Implement. +} + void clang_experimental_cas_ObjectStore_dispose(CXCASObjectStore CAS) { delete unwrap(CAS); } diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index d4a2a3fcff52f..da5e5db0ffd8d 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -110,6 +110,11 @@ void clang_experimental_DependencyScannerServiceOptions_setCASDatabases( unwrap(Opts)->Cache = DBs.Cache; } +void clang_experimental_DependencyScannerServiceOptions_setCASOptions( + CXDependencyScannerServiceOptions Opts, CXCASOptions CASOpts) { + unwrap(Opts)->CASOpts = *cas::unwrap(CASOpts); +} + void clang_experimental_DependencyScannerServiceOptions_setObjectStore( CXDependencyScannerServiceOptions Opts, CXCASObjectStore CAS) { unwrap(Opts)->CAS = cas::unwrap(CAS)->CAS; diff --git a/clang/tools/libclang/CMakeLists.txt b/clang/tools/libclang/CMakeLists.txt index d78953da68d8d..f31ec6180da23 100644 --- a/clang/tools/libclang/CMakeLists.txt +++ b/clang/tools/libclang/CMakeLists.txt @@ -35,6 +35,7 @@ set(SOURCES CRefactor.cpp CXComment.cpp CXCursor.cpp + CXError.cpp CXExtractAPI.cpp CXIndexDataConsumer.cpp CXCompilationDatabase.cpp diff --git a/clang/tools/libclang/CXError.cpp b/clang/tools/libclang/CXError.cpp new file mode 100644 index 0000000000000..ab13f8a80905f --- /dev/null +++ b/clang/tools/libclang/CXError.cpp @@ -0,0 +1,42 @@ +//===- CXError.cpp - Routines for manipulating CXErrors -------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "CXError.h" +#include "llvm/Support/CBindingWrapping.h" + +using namespace clang; +using llvm::Error; + +namespace { + +struct WrappedError { + CXErrorCode Code; + std::string Description; +}; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(WrappedError, CXError) + +} // namespace + +CXError cxerror::create(Error E, CXErrorCode Code) { + if (E) + return wrap(new WrappedError{Code, llvm::toString(std::move(E))}); + return nullptr; +} + +CXError cxerror::create(llvm::StringRef ErrorDescription, CXErrorCode Code) { + return wrap(new WrappedError{Code, std::string(ErrorDescription)}); +} + +enum CXErrorCode clang_Error_getCode(CXError CE) { return unwrap(CE)->Code; } + +const char *clang_Error_getDescription(CXError CE) { + return unwrap(CE)->Description.c_str(); +} + +void clang_Error_dispose(CXError CE) { delete unwrap(CE); } diff --git a/clang/tools/libclang/CXError.h b/clang/tools/libclang/CXError.h new file mode 100644 index 0000000000000..c0aa35247e26c --- /dev/null +++ b/clang/tools/libclang/CXError.h @@ -0,0 +1,23 @@ +//===- CXError.h - Routines for manipulating CXErrors ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_LIBCLANG_CXERROR_H +#define LLVM_CLANG_TOOLS_LIBCLANG_CXERROR_H + +#include "clang-c/CXErrorCode.h" +#include "llvm/Support/Error.h" + +namespace clang::cxerror { + +CXError create(llvm::Error E, CXErrorCode Code = CXError_Failure); +CXError create(llvm::StringRef ErrorDescription, + CXErrorCode Code = CXError_Failure); + +} // namespace clang::cxerror + +#endif diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index 1b7035d732ee1..6455177236778 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -471,9 +471,25 @@ LLVM_13 { LLVM_16 { global: + clang_Error_dispose; + clang_Error_getCode; + clang_Error_getDescription; clang_experimental_cas_ActionCache_dispose; + clang_experimental_cas_CachedCompilation_dispose; + clang_experimental_cas_CachedCompilation_getNumOutputs; + clang_experimental_cas_CachedCompilation_getOutputCASIDString; + clang_experimental_cas_CachedCompilation_getOutputName; + clang_experimental_cas_CachedCompilation_isOutputMaterialized; + clang_experimental_cas_CachedCompilation_makeGlobal; + clang_experimental_cas_CancellationToken_cancel; + clang_experimental_cas_CancellationToken_dispose; + clang_experimental_cas_CASObject_dispose; clang_experimental_cas_Databases_create; clang_experimental_cas_Databases_dispose; + clang_experimental_cas_getCachedCompilation; + clang_experimental_cas_getCachedCompilation_async; + clang_experimental_cas_loadObjectByString; + clang_experimental_cas_loadObjectByString_async; clang_experimental_cas_ObjectStore_dispose; clang_experimental_cas_OnDiskActionCache_create; clang_experimental_cas_OnDiskObjectStore_create; @@ -482,11 +498,15 @@ LLVM_16 { clang_experimental_cas_Options_setOnDiskPath; clang_experimental_cas_Options_setPluginOption; clang_experimental_cas_Options_setPluginPath; + clang_experimental_cas_replayCompilation; + clang_experimental_cas_ReplayResult_dispose; + clang_experimental_cas_ReplayResult_getStderr; clang_experimental_DependencyScannerService_create_v1; clang_experimental_DependencyScannerServiceOptions_create; clang_experimental_DependencyScannerServiceOptions_dispose; clang_experimental_DependencyScannerServiceOptions_setActionCache; clang_experimental_DependencyScannerServiceOptions_setCASDatabases; + clang_experimental_DependencyScannerServiceOptions_setCASOptions; clang_experimental_DependencyScannerServiceOptions_setDependencyMode; clang_experimental_DependencyScannerServiceOptions_setObjectStore; clang_experimental_DependencyScannerWorker_getDepGraph; diff --git a/llvm/tools/libCASPluginTest/libCASPluginTest.cpp b/llvm/tools/libCASPluginTest/libCASPluginTest.cpp index 6245fd0986ed9..b7bfde7f21a0b 100644 --- a/llvm/tools/libCASPluginTest/libCASPluginTest.cpp +++ b/llvm/tools/libCASPluginTest/libCASPluginTest.cpp @@ -52,6 +52,7 @@ struct CASPluginOptions { std::string FirstPrefix; std::string SecondPrefix; bool SimulateMissingObjects = false; + bool Logging = true; Error setOption(StringRef Name, StringRef Value); }; @@ -69,6 +70,8 @@ Error CASPluginOptions::setOption(StringRef Name, StringRef Value) { UpstreamPath = Value; else if (Name == "simulate-missing-objects") SimulateMissingObjects = true; + else if (Name == "no-logging") + Logging = false; else return createStringError(errc::invalid_argument, Twine("unknown option: ") + Name); @@ -104,6 +107,7 @@ struct CASWrapper { std::string SecondPrefix; /// If true, asynchronous "download" of an object will treat it as missing. bool SimulateMissingObjects = false; + bool Logging = true; std::unique_ptr DB; /// Used for testing the \c globally parameter of action cache APIs. Simulates /// "uploading"/"downloading" objects from/to the primary on-disk path. @@ -128,6 +132,13 @@ struct CASWrapper { /// Synchronized access to \c llvm::errs(). void syncErrs(llvm::function_ref Fn) { + if (!Logging) { + // Ignore log output. + SmallString<32> Buf; + raw_svector_ostream OS(Buf); + Fn(OS); + return; + } std::unique_lock LockGuard(Lock); Fn(errs()); errs().flush(); @@ -247,8 +258,13 @@ CASWrapper::downstreamKey(ArrayRef Key) { if (!UpstreamValue) return std::nullopt; - return DB->getGraphDB().getReference( + ObjectID Value = DB->getGraphDB().getReference( UpstreamDB->getGraphDB().getDigest(*UpstreamValue)); + Expected PutValue = DB->KVPut(Key, Value); + if (!PutValue) + return PutValue.takeError(); + assert(*PutValue == Value); + return PutValue; } llcas_cas_t llcas_cas_create(llcas_cas_options_t c_opts, char **error) { @@ -269,8 +285,8 @@ llcas_cas_t llcas_cas_create(llcas_cas_options_t c_opts, char **error) { } return wrap(new CASWrapper{Opts.FirstPrefix, Opts.SecondPrefix, - Opts.SimulateMissingObjects, std::move(*DB), - std::move(UpstreamDB)}); + Opts.SimulateMissingObjects, Opts.Logging, + std::move(*DB), std::move(UpstreamDB)}); } void llcas_cas_dispose(llcas_cas_t c_cas) { delete unwrap(c_cas); } @@ -502,11 +518,15 @@ void llcas_actioncache_get_for_digest_async(llcas_cas_t c_cas, llcas_digest_t c_key, bool globally, void *ctx_cb, llcas_actioncache_get_cb cb) { + ArrayRef Key(c_key.data, c_key.size); + SmallVector KeyBuf(Key); + unwrap(c_cas)->Pool.async([=] { llcas_objectid_t c_value; char *c_err; llcas_lookup_result_t result = llcas_actioncache_get_for_digest( - c_cas, c_key, &c_value, globally, &c_err); + c_cas, llcas_digest_t{KeyBuf.data(), KeyBuf.size()}, &c_value, globally, + &c_err); cb(ctx_cb, result, c_value, c_err); }); } @@ -539,10 +559,14 @@ void llcas_actioncache_put_for_digest_async(llcas_cas_t c_cas, llcas_objectid_t c_value, bool globally, void *ctx_cb, llcas_actioncache_put_cb cb) { + ArrayRef Key(c_key.data, c_key.size); + SmallVector KeyBuf(Key); + unwrap(c_cas)->Pool.async([=] { char *c_err; - bool failed = llcas_actioncache_put_for_digest(c_cas, c_key, c_value, - globally, &c_err); + bool failed = llcas_actioncache_put_for_digest( + c_cas, llcas_digest_t{KeyBuf.data(), KeyBuf.size()}, c_value, globally, + &c_err); cb(ctx_cb, failed, c_err); }); }