From f6b1ecb18d6d665c3f192efbb989d92e692b4021 Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Wed, 5 Mar 2025 09:03:04 -0800 Subject: [PATCH 1/2] [clang dependency scanning] Make Current Working Directory Optimization Off by Default (#129809) https://github.com/llvm/llvm-project/pull/124786 implemented current working directory (CWD) optimization and the optimization was on by default. We have discovered that build system needs to be compatible with the CWD optimization and default off is a better behavior. The build system needs to be aware that the current working directory is ignored. Without a good way of notifying the build system, it is less risky to default to off. This PR implement the change. rdar://145860213 (cherry picked from commit 7bd492ffe84b549bc0f3b26e3aab973fa81069a5) Conflicts: clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h --- .../DependencyScanning/DependencyScanningService.h | 6 +++++- clang/test/ClangScanDeps/modules-context-hash-cwd.c | 10 +++++----- clang/test/ClangScanDeps/modules-debug-dir.c | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h index af941231485e1..0715feb9fb9b9 100644 --- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h +++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h @@ -83,7 +83,11 @@ enum class ScanningOptimizations { IgnoreCWD = (1 << 4), DSS_LAST_BITMASK_ENUM(IgnoreCWD), - Default = All, + + // The build system needs to be aware that the current working + // directory is ignored. Without a good way of notifying the build + // system, it is less risky to default to off. + Default = All & (~IgnoreCWD), FullIncludeTreeIrrelevant = HeaderSearch | VFS, }; diff --git a/clang/test/ClangScanDeps/modules-context-hash-cwd.c b/clang/test/ClangScanDeps/modules-context-hash-cwd.c index 459d2c90debe6..c609a7dcbc80e 100644 --- a/clang/test/ClangScanDeps/modules-context-hash-cwd.c +++ b/clang/test/ClangScanDeps/modules-context-hash-cwd.c @@ -9,14 +9,14 @@ // RUN: sed -e "s|DIR|%/t|g" %t/cdb3.json.in > %t/cdb3.json // RUN: sed -e "s|DIR|%/t|g" %t/cdb4.json.in > %t/cdb4.json // RUN: sed -e "s|DIR|%/t|g" %t/cdb5.json.in > %t/cdb5.json -// RUN: clang-scan-deps -compilation-database %t/cdb0.json -format experimental-full > %t/result0.json -// RUN: clang-scan-deps -compilation-database %t/cdb1.json -format experimental-full > %t/result1.json +// RUN: clang-scan-deps -compilation-database %t/cdb0.json -format experimental-full -optimize-args=all > %t/result0.json +// RUN: clang-scan-deps -compilation-database %t/cdb1.json -format experimental-full -optimize-args=all > %t/result1.json // It is not a typo to use cdb1.json for result2. We intend to use the same // compilation database, but different clang-scan-deps optimize-args options. // RUN: clang-scan-deps -compilation-database %t/cdb1.json -format experimental-full -optimize-args=header-search,system-warnings,vfs,canonicalize-macros > %t/result2.json -// RUN: clang-scan-deps -compilation-database %t/cdb3.json -format experimental-full > %t/result3.json -// RUN: clang-scan-deps -compilation-database %t/cdb4.json -format experimental-full > %t/result4.json -// RUN: clang-scan-deps -compilation-database %t/cdb5.json -format experimental-full > %t/result5.json +// RUN: clang-scan-deps -compilation-database %t/cdb3.json -format experimental-full -optimize-args=all > %t/result3.json +// RUN: clang-scan-deps -compilation-database %t/cdb4.json -format experimental-full -optimize-args=all > %t/result4.json +// RUN: clang-scan-deps -compilation-database %t/cdb5.json -format experimental-full -optimize-args=all > %t/result5.json // RUN: cat %t/result0.json %t/result1.json | FileCheck %s // RUN: cat %t/result0.json %t/result2.json | FileCheck %s -check-prefix=SKIPOPT // RUN: cat %t/result3.json %t/result4.json | FileCheck %s -check-prefix=RELPATH diff --git a/clang/test/ClangScanDeps/modules-debug-dir.c b/clang/test/ClangScanDeps/modules-debug-dir.c index fadec1ad52e35..c4fb4982ed791 100644 --- a/clang/test/ClangScanDeps/modules-debug-dir.c +++ b/clang/test/ClangScanDeps/modules-debug-dir.c @@ -4,7 +4,7 @@ // RUN: split-file %s %t // RUN: sed -e "s|DIR|%/t|g" %t/cdb.json.in > %t/cdb.json // RUN: clang-scan-deps -compilation-database %t/cdb.json -format \ -// RUN: experimental-full > %t/result.json +// RUN: experimental-full -optimize-args=all > %t/result.json // RUN: cat %t/result.json | sed 's:\\\\\?:/:g' | FileCheck %s //--- cdb.json.in From 4cd29f72d8de5757dcb84af523e990fe02e16b1d Mon Sep 17 00:00:00 2001 From: Qiongsi Wu Date: Tue, 11 Mar 2025 10:01:09 -0700 Subject: [PATCH 2/2] [clang dependency scanning] C APIs for Current Working Directory Optimization (#10146) This PR implements two new C APIs so the build system can communicate with the dependency scanner about current working directory optimization. Two new functions are added: 1. `void clang_experimental_DependencyScannerServiceOptions_setCWDOptimization(CXDependencyScannerServiceOptions Opts, int)`: the caller can use this function to set `Opts` to indicate that it can support current working directory optimization. 2. `int clang_experimental_DepGraphModule_isCWDIgnored(CXDepGraphModule)`: this function can retrieve the module info from the scanner, indicating if the current working directory is ignored for this module's context hash. As an example usage, the user of the APIs (e.g. a build system) can use them to set reasonable debug working directories for each `pcm`s. rdar://145860213 (cherry picked from commit b9c866833c7fcdf7d61f286d0e56cdf1d4e7bf21) Conflicts: clang/test/ClangScanDeps/cas-fs-multiple-commands.c clang/test/ClangScanDeps/include-tree-multiple-commands.c clang/tools/libclang/libclang.map --- clang/include/clang-c/Dependencies.h | 21 ++++++++++ .../DependencyScanning/ModuleDepCollector.h | 3 ++ .../DependencyScanning/ModuleDepCollector.cpp | 1 + clang/test/ClangScanDeps/cwd-c-api.c | 42 +++++++++++++++++++ clang/test/Index/Core/scan-deps-by-mod-name.m | 1 + clang/test/Index/Core/scan-deps-cas.m | 2 + clang/test/Index/Core/scan-deps.m | 1 + clang/tools/c-index-test/core_main.cpp | 12 +++++- clang/tools/libclang/CDependencies.cpp | 17 +++++++- clang/tools/libclang/libclang.map | 6 +++ 10 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 clang/test/ClangScanDeps/cwd-c-api.c diff --git a/clang/include/clang-c/Dependencies.h b/clang/include/clang-c/Dependencies.h index f612afe837229..b90a123fd2e32 100644 --- a/clang/include/clang-c/Dependencies.h +++ b/clang/include/clang-c/Dependencies.h @@ -218,6 +218,20 @@ CINDEX_LINKAGE void clang_experimental_DependencyScannerServiceOptions_setCASOptions( CXDependencyScannerServiceOptions Opts, CXCASOptions); +/** + * Set the working directory optimization option. + * The dependency scanner service option Opts will indicate to the scanner that + * the current working directory can or cannot be ignored when computing the + * pcms' context hashes. The scanner will then determine if it is safe to + * optimize each module and act accordingly. + * + * \param Value If it is non zero, the option is on. Otherwise the + * option is off. + */ +CINDEX_LINKAGE void +clang_experimental_DependencyScannerServiceOptions_setCWDOptimization( + CXDependencyScannerServiceOptions Opts, int Value); + /** * Specify a \c CXCASObjectStore in the given options. If an object store and * action cache are available, the scanner will produce cached commands. @@ -578,6 +592,13 @@ CINDEX_LINKAGE const char * CINDEX_LINKAGE const char *clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule); +/** + * \returns 1 if the scanner ignores the current working directory when + * computing the module's context hash. Otherwise returns 0. + */ +CINDEX_LINKAGE +int clang_experimental_DepGraphModule_isCWDIgnored(CXDepGraphModule); + /** * \returns the number \c CXDepGraphTUCommand objects in the graph. */ diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h index 9179b77ae35f6..e8c7e2cd53439 100644 --- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h +++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h @@ -117,6 +117,9 @@ struct ModuleDeps { /// Whether this is a "system" module. bool IsSystem; + /// Whether current working directory is ignored. + bool IgnoreCWD; + /// The path to the modulemap file which defines this module. /// /// This can be used to explicitly build this module. This file will diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index ae1d2a3fcf5e8..6c5743b480596 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -877,6 +877,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { MD.ModuleCacheKey = Key->toString(); } + MD.IgnoreCWD = IgnoreCWD; MDC.associateWithContextHash(CI, IgnoreCWD, MD); // Finish the compiler invocation. Requires dependencies and the context hash. diff --git a/clang/test/ClangScanDeps/cwd-c-api.c b/clang/test/ClangScanDeps/cwd-c-api.c new file mode 100644 index 0000000000000..c1ecca6e8d426 --- /dev/null +++ b/clang/test/ClangScanDeps/cwd-c-api.c @@ -0,0 +1,42 @@ +// Test the current working directory C APIs. + +// RUN: rm -rf %t +// RUN: split-file %s %t + +// RUN: c-index-test core -scan-deps -working-dir %S -- %clang \ +// RUN: -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache \ +// RUN: 2>&1 > %t/no_cwd_opt.txt +// RUN: cat %t/no_cwd_opt.txt | FileCheck %s --check-prefix=NO-CWD-OPT + + +// RUN: c-index-test core -scan-deps -working-dir %S -optimize-cwd -- \ +// RUN: %clang \ +// RUN: -c %t/main.c -fmodules -fmodules-cache-path=%t/module-cache \ +// RUN: 2>&1 > %t/cwd_opt.txt +// RUN: cat %t/cwd_opt.txt | FileCheck %s --check-prefix=CWD-OPT + +//--- module.modulemap +module Mod { header "Mod.h" } + +//--- Mod.h +int foo(); + +//--- main.c +#include "Mod.h" + +int main() { + return foo(); +} + +// NO-CWD-OPT: modules: +// NO-CWD-OPT-NEXT: module: +// NO-CWD-OPT-NEXT: name: Mod +// NO-CWD-OPT-NEXT: context-hash:{{.*}} +// NO-CWD-OPT-NEXT: cwd-ignored: 0 + + +// CWD-OPT: modules: +// CWD-OPT-NEXT: module: +// CWD-OPT-NEXT: name: Mod +// CWD-OPT-NEXT: context-hash:{{.*}} +// CWD-OPT-NEXT: cwd-ignored: 1 diff --git a/clang/test/Index/Core/scan-deps-by-mod-name.m b/clang/test/Index/Core/scan-deps-by-mod-name.m index 114b059355dff..805c7069029b1 100644 --- a/clang/test/Index/Core/scan-deps-by-mod-name.m +++ b/clang/test/Index/Core/scan-deps-by-mod-name.m @@ -13,6 +13,7 @@ // CHECK-NEXT: module: // CHECK-NEXT: name: ModA // CHECK-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] +// CHECK-NEXT: cwd-ignored: 0 // CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap // CHECK-NEXT: module-deps: // CHECK-NEXT: file-deps: diff --git a/clang/test/Index/Core/scan-deps-cas.m b/clang/test/Index/Core/scan-deps-cas.m index 4ec74d26e511f..90d9a69bbda55 100644 --- a/clang/test/Index/Core/scan-deps-cas.m +++ b/clang/test/Index/Core/scan-deps-cas.m @@ -32,6 +32,7 @@ // CHECK-NEXT: module: // CHECK-NEXT: name: ModA // CHECK-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] +// CHECK-NEXT: cwd-ignored: 0 // CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap // CHECK-NEXT: cache-key: [[CASFS_MODA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] // CHECK-NEXT: module-deps: @@ -70,6 +71,7 @@ // INCLUDE_TREE-NEXT: module: // INCLUDE_TREE-NEXT: name: ModA // INCLUDE_TREE-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] +// INCLUDE_TREE-NEXT: cwd-ignored: 0 // INCLUDE_TREE-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap // INCLUDE_TREE-NEXT: include-tree-id: [[ModA_INCLUDE_TREE_ID:llvmcas://[[:xdigit:]]+]] // INCLUDE_TREE-NEXT: cache-key: [[ModA_CACHE_KEY:llvmcas://[[:xdigit:]]+]] diff --git a/clang/test/Index/Core/scan-deps.m b/clang/test/Index/Core/scan-deps.m index 04aada3ba9507..f9dd2c4d07152 100644 --- a/clang/test/Index/Core/scan-deps.m +++ b/clang/test/Index/Core/scan-deps.m @@ -21,6 +21,7 @@ // CHECK-NEXT: module: // CHECK-NEXT: name: ModA // CHECK-NEXT: context-hash: [[HASH_MOD_A:[A-Z0-9]+]] +// CHECK-NEXT: cwd-ignored: 0 // CHECK-NEXT: module-map-path: [[PREFIX]]/Inputs/module/module.modulemap // CHECK-NEXT: module-deps: // CHECK-NEXT: file-deps: diff --git a/clang/tools/c-index-test/core_main.cpp b/clang/tools/c-index-test/core_main.cpp index 04c6516830011..bc1eb999b6cfe 100644 --- a/clang/tools/c-index-test/core_main.cpp +++ b/clang/tools/c-index-test/core_main.cpp @@ -161,7 +161,11 @@ static cl::opt TestCASCancellation( "test-cas-cancellation", cl::desc( "perform extra CAS API invocation and cancel it for testing purposes")); -} +static cl::opt OptimizeCWD( + "optimize-cwd", + cl::desc( + "instruct the scanner to ignore current working directory if safe.")); +} // namespace options } // anonymous namespace static void printSymbolInfo(SymbolInfo SymInfo, raw_ostream &OS); @@ -714,6 +718,10 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, clang_experimental_DependencyScannerServiceOptions_setDependencyMode( Opts, CXDependencyMode_Full); + if (options::OptimizeCWD) + clang_experimental_DependencyScannerServiceOptions_setCWDOptimization(Opts, + 1); + if (DBs) clang_experimental_DependencyScannerServiceOptions_setCASDatabases(Opts, DBs); @@ -806,6 +814,7 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, const char *Name = clang_experimental_DepGraphModule_getName(Mod); const char *ContextHash = clang_experimental_DepGraphModule_getContextHash(Mod); + int CwdIgnored = clang_experimental_DepGraphModule_isCWDIgnored(Mod); const char *ModuleMapPath = clang_experimental_DepGraphModule_getModuleMapPath(Mod); const char *ModuleIncludeTreeID = @@ -823,6 +832,7 @@ static int scanDeps(ArrayRef Args, std::string WorkingDirectory, llvm::outs() << " module:\n" << " name: " << Name << "\n" << " context-hash: " << ContextHash << "\n" + << " cwd-ignored: " << CwdIgnored << "\n" << " module-map-path: " << (ModuleMapPath ? ModuleMapPath : "") << "\n"; if (ModuleIncludeTreeID) diff --git a/clang/tools/libclang/CDependencies.cpp b/clang/tools/libclang/CDependencies.cpp index ac66d0a42b71c..488494b187cea 100644 --- a/clang/tools/libclang/CDependencies.cpp +++ b/clang/tools/libclang/CDependencies.cpp @@ -34,6 +34,7 @@ namespace { struct DependencyScannerServiceOptions { ScanningOutputFormat ConfiguredFormat = ScanningOutputFormat::Full; CASOptions CASOpts; + ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default; std::shared_ptr CAS; std::shared_ptr Cache; @@ -115,6 +116,15 @@ void clang_experimental_DependencyScannerServiceOptions_setCASOptions( unwrap(Opts)->CASOpts = *cas::unwrap(CASOpts); } +void clang_experimental_DependencyScannerServiceOptions_setCWDOptimization( + CXDependencyScannerServiceOptions Opts, int Value) { + auto Mask = + Value != 0 ? ScanningOptimizations::All : ScanningOptimizations::None; + auto OptArgs = unwrap(Opts)->OptimizeArgs; + unwrap(Opts)->OptimizeArgs = (OptArgs & ~ScanningOptimizations::IgnoreCWD) | + (ScanningOptimizations::IgnoreCWD & Mask); +} + void clang_experimental_DependencyScannerServiceOptions_setObjectStore( CXDependencyScannerServiceOptions Opts, CXCASObjectStore CAS) { unwrap(Opts)->CAS = cas::unwrap(CAS)->CAS; @@ -171,7 +181,8 @@ clang_experimental_DependencyScannerService_create_v1( } return wrap(new DependencyScanningService( ScanningMode::DependencyDirectivesScan, Format, unwrap(Opts)->CASOpts, - std::move(CAS), std::move(Cache), std::move(FS))); + std::move(CAS), std::move(Cache), std::move(FS), + unwrap(Opts)->OptimizeArgs)); } void clang_experimental_DependencyScannerService_dispose_v0( @@ -630,6 +641,10 @@ clang_experimental_DepGraphModule_getCacheKey(CXDepGraphModule CXDepMod) { return nullptr; } +int clang_experimental_DepGraphModule_isCWDIgnored(CXDepGraphModule CXDepMod) { + return unwrap(CXDepMod)->ModDeps->IgnoreCWD; +} + size_t clang_experimental_DepGraph_getNumTUCommands(CXDepGraph Graph) { TranslationUnitDeps &TUDeps = unwrap(Graph)->TUDeps; return TUDeps.Commands.size(); diff --git a/clang/tools/libclang/libclang.map b/clang/tools/libclang/libclang.map index a78239b6b7de4..b6bb245d7dad0 100644 --- a/clang/tools/libclang/libclang.map +++ b/clang/tools/libclang/libclang.map @@ -569,6 +569,12 @@ LLVM_19 { clang_Cursor_getBinaryOpcodeStr; }; +LLVM_21 { + global: + clang_experimental_DependencyScannerServiceOptions_setCWDOptimization; + clang_experimental_DepGraphModule_isCWDIgnored; +}; + # Example of how to add a new symbol version entry. If you do add a new symbol # version, please update the example to depend on the version you added. # LLVM_X {