Skip to content

Commit

Permalink
[clang][modules] Avoid storing command-line macro definitions into im…
Browse files Browse the repository at this point in the history 
…plicitly built PCM files

With implicit modules, it's impossible to load a PCM file that was built using different command-line macro definitions. This is guaranteed by the fact that they contribute to the context hash. This means that we don't need to store those macros into PCM files for validation purposes. This patch avoids serializing them in those circumstances, since there's no other use for command-line macro definitions (besides "-module-file-info").

For a typical Apple project, this speeds up the dependency scan by 5.6% and shrinks the cache with scanning PCMs by 26%.

Reviewed By: benlangmuir

Differential Revision: https://reviews.llvm.org/D158136
  • Loading branch information
@jansvoboda11
jansvoboda11 committed Aug 17, 2023
1 parent 2041611 commit 6a11557
Show file tree
Hide file tree
Showing 8 changed files with 148 additions and 121 deletions.
10 changes: 6 additions & 4 deletions clang/include/clang/Serialization/ASTReader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ class ASTReaderListener {
/// \returns true to indicate the preprocessor options are invalid, or false
/// otherwise.
virtual bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool Complain,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) {
return false;
}
Expand Down Expand Up @@ -274,7 +274,7 @@ class ChainedASTReaderListener : public ASTReaderListener {
StringRef SpecificModuleCachePath,
bool Complain) override;
bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool Complain,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) override;

void ReadCounter(const serialization::ModuleFile &M, unsigned Value) override;
Expand Down Expand Up @@ -304,7 +304,8 @@ class PCHValidator : public ASTReaderListener {
bool AllowCompatibleDifferences) override;
bool ReadDiagnosticOptions(IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts,
bool Complain) override;
bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts, bool Complain,
bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) override;
bool ReadHeaderSearchOptions(const HeaderSearchOptions &HSOpts,
StringRef SpecificModuleCachePath,
Expand All @@ -325,7 +326,8 @@ class SimpleASTReaderListener : public ASTReaderListener {
public:
SimpleASTReaderListener(Preprocessor &PP) : PP(PP) {}

bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts, bool Complain,
bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) override;
};

Expand Down
8 changes: 7 additions & 1 deletion clang/include/clang/Serialization/ASTWriter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,11 @@ class ASTWriter : public ASTDeserializationListener,
/// file is up to date, but not otherwise.
bool IncludeTimestamps;

/// Indicates whether the AST file being written is an implicit module.
/// If that's the case, we may be able to skip writing some information that
/// are guaranteed to be the same in the importer by the context hash.
bool BuildingImplicitModule = false;

/// Indicates when the AST writing is actively performing
/// serialization, rather than just queueing updates.
bool WritingAST = false;
Expand Down Expand Up @@ -571,7 +576,7 @@ class ASTWriter : public ASTDeserializationListener,
ASTWriter(llvm::BitstreamWriter &Stream, SmallVectorImpl<char> &Buffer,
InMemoryModuleCache &ModuleCache,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool IncludeTimestamps = true);
bool IncludeTimestamps = true, bool BuildingImplicitModule = false);
~ASTWriter() override;

ASTContext &getASTContext() const {
Expand Down Expand Up @@ -809,6 +814,7 @@ class PCHGenerator : public SemaConsumer {
std::shared_ptr<PCHBuffer> Buffer,
ArrayRef<std::shared_ptr<ModuleFileExtension>> Extensions,
bool AllowASTWithErrors = false, bool IncludeTimestamps = true,
bool BuildingImplicitModule = false,
bool ShouldCacheASTInMemory = false);
~PCHGenerator() override;

Expand Down
3 changes: 2 additions & 1 deletion clang/lib/Frontend/ASTUnit.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,7 +581,8 @@ class ASTInfoCollector : public ASTReaderListener {
return false;
}

bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts, bool Complain,
bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) override {
this->PPOpts = PPOpts;
return false;
Expand Down
8 changes: 5 additions & 3 deletions clang/lib/Frontend/FrontendActions.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
CI.getPreprocessor(), CI.getModuleCache(), OutputFile, Sysroot, Buffer,
FrontendOpts.ModuleFileExtensions,
CI.getPreprocessorOpts().AllowPCHWithCompilerErrors,
FrontendOpts.IncludeTimestamps, +CI.getLangOpts().CacheGeneratedPCH));
FrontendOpts.IncludeTimestamps, FrontendOpts.BuildingImplicitModule,
+CI.getLangOpts().CacheGeneratedPCH));
Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator(
CI, std::string(InFile), OutputFile, std::move(OS), Buffer));

Expand Down Expand Up @@ -204,6 +205,7 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI,
/*IncludeTimestamps=*/
+CI.getFrontendOpts().BuildingImplicitModule &&
+CI.getFrontendOpts().IncludeTimestamps,
/*BuildingImplicitModule=*/+CI.getFrontendOpts().BuildingImplicitModule,
/*ShouldCacheASTInMemory=*/
+CI.getFrontendOpts().BuildingImplicitModule));
Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator(
Expand Down Expand Up @@ -660,15 +662,15 @@ namespace {
}

bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool Complain,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) override {
Out.indent(2) << "Preprocessor options:\n";
DUMP_BOOLEAN(PPOpts.UsePredefines,
"Uses compiler/target-specific predefines [-undef]");
DUMP_BOOLEAN(PPOpts.DetailedRecord,
"Uses detailed preprocessing record (for indexing)");

if (!PPOpts.Macros.empty()) {
if (ReadMacros) {
Out.indent(4) << "Predefined macros:\n";
}

Expand Down
193 changes: 99 additions & 94 deletions clang/lib/Serialization/ASTReader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,12 @@ bool ChainedASTReaderListener::ReadHeaderSearchOptions(
}

bool ChainedASTReaderListener::ReadPreprocessorOptions(
const PreprocessorOptions &PPOpts, bool Complain,
const PreprocessorOptions &PPOpts, bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) {
return First->ReadPreprocessorOptions(PPOpts, Complain,
return First->ReadPreprocessorOptions(PPOpts, ReadMacros, Complain,
SuggestedPredefines) ||
Second->ReadPreprocessorOptions(PPOpts, Complain, SuggestedPredefines);
Second->ReadPreprocessorOptions(PPOpts, ReadMacros, Complain,
SuggestedPredefines);
}

void ChainedASTReaderListener::ReadCounter(const serialization::ModuleFile &M,
Expand Down Expand Up @@ -658,92 +659,95 @@ enum OptionValidation {
/// are no differences in the options between the two.
static bool checkPreprocessorOptions(
const PreprocessorOptions &PPOpts,
const PreprocessorOptions &ExistingPPOpts, DiagnosticsEngine *Diags,
FileManager &FileMgr, std::string &SuggestedPredefines,
const LangOptions &LangOpts,
const PreprocessorOptions &ExistingPPOpts, bool ReadMacros,
DiagnosticsEngine *Diags, FileManager &FileMgr,
std::string &SuggestedPredefines, const LangOptions &LangOpts,
OptionValidation Validation = OptionValidateContradictions) {
// Check macro definitions.
MacroDefinitionsMap ASTFileMacros;
collectMacroDefinitions(PPOpts, ASTFileMacros);
MacroDefinitionsMap ExistingMacros;
SmallVector<StringRef, 4> ExistingMacroNames;
collectMacroDefinitions(ExistingPPOpts, ExistingMacros, &ExistingMacroNames);

// Use a line marker to enter the <command line> file, as the defines and
// undefines here will have come from the command line.
SuggestedPredefines += "# 1 \"<command line>\" 1\n";

for (unsigned I = 0, N = ExistingMacroNames.size(); I != N; ++I) {
// Dig out the macro definition in the existing preprocessor options.
StringRef MacroName = ExistingMacroNames[I];
std::pair<StringRef, bool> Existing = ExistingMacros[MacroName];

// Check whether we know anything about this macro name or not.
llvm::StringMap<std::pair<StringRef, bool /*IsUndef*/>>::iterator Known =
ASTFileMacros.find(MacroName);
if (Validation == OptionValidateNone || Known == ASTFileMacros.end()) {
if (Validation == OptionValidateStrictMatches) {
// If strict matches are requested, don't tolerate any extra defines on
// the command line that are missing in the AST file.
if (ReadMacros) {
// Check macro definitions.
MacroDefinitionsMap ASTFileMacros;
collectMacroDefinitions(PPOpts, ASTFileMacros);
MacroDefinitionsMap ExistingMacros;
SmallVector<StringRef, 4> ExistingMacroNames;
collectMacroDefinitions(ExistingPPOpts, ExistingMacros,
&ExistingMacroNames);

// Use a line marker to enter the <command line> file, as the defines and
// undefines here will have come from the command line.
SuggestedPredefines += "# 1 \"<command line>\" 1\n";

for (unsigned I = 0, N = ExistingMacroNames.size(); I != N; ++I) {
// Dig out the macro definition in the existing preprocessor options.
StringRef MacroName = ExistingMacroNames[I];
std::pair<StringRef, bool> Existing = ExistingMacros[MacroName];

// Check whether we know anything about this macro name or not.
llvm::StringMap<std::pair<StringRef, bool /*IsUndef*/>>::iterator Known =
ASTFileMacros.find(MacroName);
if (Validation == OptionValidateNone || Known == ASTFileMacros.end()) {
if (Validation == OptionValidateStrictMatches) {
// If strict matches are requested, don't tolerate any extra defines
// on the command line that are missing in the AST file.
if (Diags) {
Diags->Report(diag::err_pch_macro_def_undef) << MacroName << true;
}
return true;
}
// FIXME: Check whether this identifier was referenced anywhere in the
// AST file. If so, we should reject the AST file. Unfortunately, this
// information isn't in the control block. What shall we do about it?

if (Existing.second) {
SuggestedPredefines += "#undef ";
SuggestedPredefines += MacroName.str();
SuggestedPredefines += '\n';
} else {
SuggestedPredefines += "#define ";
SuggestedPredefines += MacroName.str();
SuggestedPredefines += ' ';
SuggestedPredefines += Existing.first.str();
SuggestedPredefines += '\n';
}
continue;
}

// If the macro was defined in one but undef'd in the other, we have a
// conflict.
if (Existing.second != Known->second.second) {
if (Diags) {
Diags->Report(diag::err_pch_macro_def_undef) << MacroName << true;
Diags->Report(diag::err_pch_macro_def_undef)
<< MacroName << Known->second.second;
}
return true;
}
// FIXME: Check whether this identifier was referenced anywhere in the
// AST file. If so, we should reject the AST file. Unfortunately, this
// information isn't in the control block. What shall we do about it?

if (Existing.second) {
SuggestedPredefines += "#undef ";
SuggestedPredefines += MacroName.str();
SuggestedPredefines += '\n';
} else {
SuggestedPredefines += "#define ";
SuggestedPredefines += MacroName.str();
SuggestedPredefines += ' ';
SuggestedPredefines += Existing.first.str();
SuggestedPredefines += '\n';

// If the macro was #undef'd in both, or if the macro bodies are
// identical, it's fine.
if (Existing.second || Existing.first == Known->second.first) {
ASTFileMacros.erase(Known);
continue;
}
continue;
}

// If the macro was defined in one but undef'd in the other, we have a
// conflict.
if (Existing.second != Known->second.second) {
// The macro bodies differ; complain.
if (Diags) {
Diags->Report(diag::err_pch_macro_def_undef)
<< MacroName << Known->second.second;
Diags->Report(diag::err_pch_macro_def_conflict)
<< MacroName << Known->second.first << Existing.first;
}
return true;
}

// If the macro was #undef'd in both, or if the macro bodies are identical,
// it's fine.
if (Existing.second || Existing.first == Known->second.first) {
ASTFileMacros.erase(Known);
continue;
}

// The macro bodies differ; complain.
if (Diags) {
Diags->Report(diag::err_pch_macro_def_conflict)
<< MacroName << Known->second.first << Existing.first;
}
return true;
}

// Leave the <command line> file and return to <built-in>.
SuggestedPredefines += "# 1 \"<built-in>\" 2\n";
// Leave the <command line> file and return to <built-in>.
SuggestedPredefines += "# 1 \"<built-in>\" 2\n";

if (Validation == OptionValidateStrictMatches) {
// If strict matches are requested, don't tolerate any extra defines in
// the AST file that are missing on the command line.
for (const auto &MacroName : ASTFileMacros.keys()) {
if (Diags) {
Diags->Report(diag::err_pch_macro_def_undef) << MacroName << false;
if (Validation == OptionValidateStrictMatches) {
// If strict matches are requested, don't tolerate any extra defines in
// the AST file that are missing on the command line.
for (const auto &MacroName : ASTFileMacros.keys()) {
if (Diags) {
Diags->Report(diag::err_pch_macro_def_undef) << MacroName << false;
}
return true;
}
return true;
}
}

Expand Down Expand Up @@ -805,24 +809,22 @@ static bool checkPreprocessorOptions(
}

bool PCHValidator::ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool Complain,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) {
const PreprocessorOptions &ExistingPPOpts = PP.getPreprocessorOpts();

return checkPreprocessorOptions(PPOpts, ExistingPPOpts,
Complain? &Reader.Diags : nullptr,
PP.getFileManager(),
SuggestedPredefines,
PP.getLangOpts());
return checkPreprocessorOptions(
PPOpts, ExistingPPOpts, ReadMacros, Complain ? &Reader.Diags : nullptr,
PP.getFileManager(), SuggestedPredefines, PP.getLangOpts());
}

bool SimpleASTReaderListener::ReadPreprocessorOptions(
const PreprocessorOptions &PPOpts,
bool Complain,
std::string &SuggestedPredefines) {
return checkPreprocessorOptions(PPOpts, PP.getPreprocessorOpts(), nullptr,
PP.getFileManager(), SuggestedPredefines,
PP.getLangOpts(), OptionValidateNone);
const PreprocessorOptions &PPOpts, bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) {
return checkPreprocessorOptions(PPOpts, PP.getPreprocessorOpts(), ReadMacros,
nullptr, PP.getFileManager(),
SuggestedPredefines, PP.getLangOpts(),
OptionValidateNone);
}

/// Check the header search options deserialized from the control block
Expand Down Expand Up @@ -5274,10 +5276,10 @@ namespace {
}

bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
bool Complain,
bool ReadMacros, bool Complain,
std::string &SuggestedPredefines) override {
return checkPreprocessorOptions(
PPOpts, ExistingPPOpts, /*Diags=*/nullptr, FileMgr,
PPOpts, ExistingPPOpts, ReadMacros, /*Diags=*/nullptr, FileMgr,
SuggestedPredefines, ExistingLangOpts,
StrictOptionMatches ? OptionValidateStrictMatches
: OptionValidateContradictions);
Expand Down Expand Up @@ -6048,10 +6050,13 @@ bool ASTReader::ParsePreprocessorOptions(const RecordData &Record,
unsigned Idx = 0;

// Macro definitions/undefs
for (unsigned N = Record[Idx++]; N; --N) {
std::string Macro = ReadString(Record, Idx);
bool IsUndef = Record[Idx++];
PPOpts.Macros.push_back(std::make_pair(Macro, IsUndef));
bool ReadMacros = Record[Idx++];
if (ReadMacros) {
for (unsigned N = Record[Idx++]; N; --N) {
std::string Macro = ReadString(Record, Idx);
bool IsUndef = Record[Idx++];
PPOpts.Macros.push_back(std::make_pair(Macro, IsUndef));
}
}

// Includes
Expand All @@ -6070,7 +6075,7 @@ bool ASTReader::ParsePreprocessorOptions(const RecordData &Record,
PPOpts.ObjCXXARCStandardLibrary =
static_cast<ObjCXXARCStandardLibraryKind>(Record[Idx++]);
SuggestedPredefines.clear();
return Listener.ReadPreprocessorOptions(PPOpts, Complain,
return Listener.ReadPreprocessorOptions(PPOpts, ReadMacros, Complain,
SuggestedPredefines);
}

Expand Down

0 comments on commit 6a11557

Please sign in to comment.