147 changes: 147 additions & 0 deletions clang/include/clang/Serialization/ModuleFileExtension.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
//===-- ModuleFileExtension.h - Module File Extensions ----------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H
#define LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H

#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include <memory>
#include <string>

namespace llvm {
class BitstreamCursor;
class BitstreamWriter;
class hash_code;
class raw_ostream;
}

namespace clang {

class ASTReader;
class ASTWriter;

namespace serialization {
class ModuleFile;
} // end namespace serialization

/// Metadata for a module file extension.
struct ModuleFileExtensionMetadata {
/// The name used to identify this particular extension block within
/// the resulting module file. It should be unique to the particular
/// extension, because this name will be used to match the name of
/// an extension block to the appropriate reader.
std::string BlockName;

/// The major version of the extension data.
unsigned MajorVersion;

/// The minor version of the extension data.
unsigned MinorVersion;

/// A string containing additional user information that will be
/// stored with the metadata.
std::string UserInfo;
};

class ModuleFileExtensionReader;
class ModuleFileExtensionWriter;

/// An abstract superclass that describes a custom extension to the
/// module/precompiled header file format.
///
/// A module file extension can introduce additional information into
/// compiled module files (.pcm) and precompiled headers (.pch) via a
/// custom writer that can then be accessed via a custom reader when
/// the module file or precompiled header is loaded.
class ModuleFileExtension : public llvm::RefCountedBase<ModuleFileExtension> {
public:
virtual ~ModuleFileExtension();

/// Retrieves the metadata for this module file extension.
virtual ModuleFileExtensionMetadata getExtensionMetadata() const = 0;

/// Hash information about the presence of this extension into the
/// module hash code.
///
/// The module hash code is used to distinguish different variants
/// of a module that are incompatible. If the presence, absence, or
/// version of the module file extension should force the creation
/// of a separate set of module files, override this method to
/// combine that distinguishing information into the module hash
/// code.
///
/// The default implementation of this function simply returns the
/// hash code as given, so the presence/absence of this extension
/// does not distinguish module files.
virtual llvm::hash_code hashExtension(llvm::hash_code Code) const;

/// Create a new module file extension writer, which will be
/// responsible for writing the extension contents into a particular
/// module file.
virtual std::unique_ptr<ModuleFileExtensionWriter>
createExtensionWriter(ASTWriter &Writer) = 0;

/// Create a new module file extension reader, given the
/// metadata read from the block and the cursor into the extension
/// block.
///
/// May return null to indicate that an extension block with the
/// given metadata cannot be read.
virtual std::unique_ptr<ModuleFileExtensionReader>
createExtensionReader(const ModuleFileExtensionMetadata &Metadata,
ASTReader &Reader, serialization::ModuleFile &Mod,
const llvm::BitstreamCursor &Stream) = 0;
};

/// Abstract base class that writes a module file extension block into
/// a module file.
class ModuleFileExtensionWriter {
ModuleFileExtension *Extension;

protected:
ModuleFileExtensionWriter(ModuleFileExtension *Extension)
: Extension(Extension) { }

public:
virtual ~ModuleFileExtensionWriter();

/// Retrieve the module file extension with which this writer is
/// associated.
ModuleFileExtension *getExtension() const { return Extension; }

/// Write the contents of the extension block into the given bitstream.
///
/// Responsible for writing the contents of the extension into the
/// given stream. All of the contents should be written into custom
/// records with IDs >= FIRST_EXTENSION_RECORD_ID.
virtual void writeExtensionContents(llvm::BitstreamWriter &Stream) = 0;
};

/// Abstract base class that reads a module file extension block from
/// a module file.
///
/// Subclasses
class ModuleFileExtensionReader {
ModuleFileExtension *Extension;

protected:
ModuleFileExtensionReader(ModuleFileExtension *Extension)
: Extension(Extension) { }

public:
/// Retrieve the module file extension with which this reader is
/// associated.
ModuleFileExtension *getExtension() const { return Extension; }

virtual ~ModuleFileExtensionReader();
};

} // end namespace clang

#endif // LLVM_CLANG_FRONTEND_MODULEFILEEXTENSION_H
7 changes: 4 additions & 3 deletions clang/lib/Frontend/ASTUnit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ struct ASTUnit::ASTWriterData {
llvm::BitstreamWriter Stream;
ASTWriter Writer;

ASTWriterData() : Stream(Buffer), Writer(Stream) { }
ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { }
};

void ASTUnit::clearFileLevelDecls() {
Expand Down Expand Up @@ -709,7 +709,7 @@ std::unique_ptr<ASTUnit> ASTUnit::LoadFromASTFile(
bool disableValid = false;
if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION"))
disableValid = true;
AST->Reader = new ASTReader(PP, Context, PCHContainerRdr,
AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, { },
/*isysroot=*/"",
/*DisableValidation=*/disableValid,
AllowPCHWithCompilerErrors);
Expand Down Expand Up @@ -927,6 +927,7 @@ class PrecompilePreambleConsumer : public PCHGenerator {
const Preprocessor &PP, StringRef isysroot,
raw_ostream *Out)
: PCHGenerator(PP, "", nullptr, isysroot, std::make_shared<PCHBuffer>(),
ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>>(),
/*AllowASTWithErrors=*/true),
Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action),
Out(Out) {
Expand Down Expand Up @@ -2500,7 +2501,7 @@ bool ASTUnit::serialize(raw_ostream &OS) {

SmallString<128> Buffer;
llvm::BitstreamWriter Stream(Buffer);
ASTWriter Writer(Stream);
ASTWriter Writer(Stream, { });
return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS);
}

Expand Down
1 change: 1 addition & 0 deletions clang/lib/Frontend/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ add_clang_library(clangFrontend
PrintPreprocessedOutput.cpp
SerializedDiagnosticPrinter.cpp
SerializedDiagnosticReader.cpp
TestModuleFileExtension.cpp
TextDiagnostic.cpp
TextDiagnosticBuffer.cpp
TextDiagnosticPrinter.cpp
Expand Down
5 changes: 4 additions & 1 deletion clang/lib/Frontend/ChainedIncludesSource.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ createASTReader(CompilerInstance &CI, StringRef pchFile,
std::unique_ptr<ASTReader> Reader;
Reader.reset(new ASTReader(PP, CI.getASTContext(),
CI.getPCHContainerReader(),
/*Extensions=*/{ },
/*isysroot=*/"", /*DisableValidation=*/true));
for (unsigned ti = 0; ti < bufNames.size(); ++ti) {
StringRef sr(bufNames[ti]);
Expand Down Expand Up @@ -160,8 +161,10 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource(
Clang->createASTContext();

auto Buffer = std::make_shared<PCHBuffer>();
ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>> Extensions;
auto consumer = llvm::make_unique<PCHGenerator>(
Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer);
Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer,
Extensions);
Clang->getASTContext().setASTMutationListener(
consumer->GetASTMutationListener());
Clang->setASTConsumer(std::move(consumer));
Expand Down
14 changes: 9 additions & 5 deletions clang/lib/Frontend/CompilerInstance.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,9 @@ void CompilerInstance::createPCHExternalASTSource(
ModuleManager = createPCHExternalASTSource(
Path, getHeaderSearchOpts().Sysroot, DisablePCHValidation,
AllowPCHWithCompilerErrors, getPreprocessor(), getASTContext(),
getPCHContainerReader(), DeserializationListener,
getPCHContainerReader(),
getFrontendOpts().ModuleFileExtensions,
DeserializationListener,
OwnDeserializationListener, Preamble,
getFrontendOpts().UseGlobalModuleIndex);
}
Expand All @@ -417,15 +419,16 @@ IntrusiveRefCntPtr<ASTReader> CompilerInstance::createPCHExternalASTSource(
StringRef Path, StringRef Sysroot, bool DisablePCHValidation,
bool AllowPCHWithCompilerErrors, Preprocessor &PP, ASTContext &Context,
const PCHContainerReader &PCHContainerRdr,
ArrayRef<IntrusiveRefCntPtr<ModuleFileExtension>> Extensions,
void *DeserializationListener, bool OwnDeserializationListener,
bool Preamble, bool UseGlobalModuleIndex) {
HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts();

IntrusiveRefCntPtr<ASTReader> Reader(new ASTReader(
PP, Context, PCHContainerRdr, Sysroot.empty() ? "" : Sysroot.data(),
DisablePCHValidation, AllowPCHWithCompilerErrors,
/*AllowConfigurationMismatch*/ false, HSOpts.ModulesValidateSystemHeaders,
UseGlobalModuleIndex));
PP, Context, PCHContainerRdr, Extensions,
Sysroot.empty() ? "" : Sysroot.data(), DisablePCHValidation,
AllowPCHWithCompilerErrors, /*AllowConfigurationMismatch*/ false,
HSOpts.ModulesValidateSystemHeaders, UseGlobalModuleIndex));

// We need the external source to be set up before we read the AST, because
// eagerly-deserialized declarations may use it.
Expand Down Expand Up @@ -1267,6 +1270,7 @@ void CompilerInstance::createModuleManager() {
*FrontendTimerGroup);
ModuleManager = new ASTReader(
getPreprocessor(), getASTContext(), getPCHContainerReader(),
getFrontendOpts().ModuleFileExtensions,
Sysroot.empty() ? "" : Sysroot.c_str(), PPOpts.DisablePCHValidation,
/*AllowASTWithCompilerErrors=*/false,
/*AllowConfigurationMismatch=*/false,
Expand Down
52 changes: 52 additions & 0 deletions clang/lib/Frontend/CompilerInvocation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
//
//===----------------------------------------------------------------------===//

#include "TestModuleFileExtension.h"
#include "clang/Frontend/CompilerInvocation.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/Version.h"
Expand All @@ -19,6 +20,7 @@
#include "clang/Frontend/Utils.h"
#include "clang/Lex/HeaderSearchOptions.h"
#include "clang/Serialization/ASTReader.h"
#include "clang/Serialization/ModuleFileExtension.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
Expand Down Expand Up @@ -832,6 +834,30 @@ static void ParseFileSystemArgs(FileSystemOptions &Opts, ArgList &Args) {
Opts.WorkingDir = Args.getLastArgValue(OPT_working_directory);
}

/// Parse the argument to the -ftest-module-file-extension
/// command-line argument.
///
/// \returns true on error, false on success.
static bool parseTestModuleFileExtensionArg(StringRef Arg,
std::string &BlockName,
unsigned &MajorVersion,
unsigned &MinorVersion,
bool &Hashed,
std::string &UserInfo) {
SmallVector<StringRef, 5> Args;
Arg.split(Args, ':', 5);
if (Args.size() < 5)
return true;

BlockName = Args[0];
if (Args[1].getAsInteger(10, MajorVersion)) return true;
if (Args[2].getAsInteger(10, MinorVersion)) return true;
if (Args[3].getAsInteger(2, Hashed)) return true;
if (Args.size() > 4)
UserInfo = Args[4];
return false;
}

static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
DiagnosticsEngine &Diags) {
using namespace options;
Expand Down Expand Up @@ -924,6 +950,26 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
if (A->getValue(0) == Opts.AddPluginActions[i])
Opts.AddPluginArgs[i].emplace_back(A->getValue(1));

for (const std::string &Arg :
Args.getAllArgValues(OPT_ftest_module_file_extension_EQ)) {
std::string BlockName;
unsigned MajorVersion;
unsigned MinorVersion;
bool Hashed;
std::string UserInfo;
if (parseTestModuleFileExtensionArg(Arg, BlockName, MajorVersion,
MinorVersion, Hashed, UserInfo)) {
Diags.Report(diag::err_test_module_file_extension_format) << Arg;

continue;
}

// Add the testing module file extension.
Opts.ModuleFileExtensions.push_back(
new TestModuleFileExtension(BlockName, MajorVersion, MinorVersion,
Hashed, UserInfo));
}

if (const Arg *A = Args.getLastArg(OPT_code_completion_at)) {
Opts.CodeCompletionAt =
ParsedSourceLocation::FromString(A->getValue());
Expand Down Expand Up @@ -2076,6 +2122,12 @@ std::string CompilerInvocation::getModuleHash() const {
// Extend the signature with the user build path.
code = hash_combine(code, hsOpts.ModuleUserBuildPath);

// Extend the signature with the module file extensions.
const FrontendOptions &frontendOpts = getFrontendOpts();
for (auto ext : frontendOpts.ModuleFileExtensions) {
code = ext->hashExtension(code);
}

// Darwin-specific hack: if we have a sysroot, use the contents and
// modification time of
// $sysroot/System/Library/CoreServices/SystemVersion.plist
Expand Down
30 changes: 25 additions & 5 deletions clang/lib/Frontend/FrontendActions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,8 @@ GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
auto Buffer = std::make_shared<PCHBuffer>();
std::vector<std::unique_ptr<ASTConsumer>> Consumers;
Consumers.push_back(llvm::make_unique<PCHGenerator>(
CI.getPreprocessor(), OutputFile, nullptr, Sysroot, Buffer));
CI.getPreprocessor(), OutputFile, nullptr, Sysroot,
Buffer, CI.getFrontendOpts().ModuleFileExtensions));
Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator(
CI, InFile, OutputFile, OS, Buffer));

Expand Down Expand Up @@ -133,10 +134,13 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI,

auto Buffer = std::make_shared<PCHBuffer>();
std::vector<std::unique_ptr<ASTConsumer>> Consumers;

Consumers.push_back(llvm::make_unique<PCHGenerator>(
CI.getPreprocessor(), OutputFile, Module, Sysroot, Buffer,
/*AllowASTWithErrors*/false,
/*IncludeTimestamps*/+CI.getFrontendOpts().BuildingImplicitModule));
CI.getPreprocessor(), OutputFile, Module, Sysroot,
Buffer, CI.getFrontendOpts().ModuleFileExtensions,
/*AllowASTWithErrors=*/false,
/*IncludeTimestamps=*/
+CI.getFrontendOpts().BuildingImplicitModule));
Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator(
CI, InFile, OutputFile, OS, Buffer));
return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
Expand Down Expand Up @@ -421,6 +425,7 @@ void VerifyPCHAction::ExecuteAction() {
const std::string &Sysroot = CI.getHeaderSearchOpts().Sysroot;
std::unique_ptr<ASTReader> Reader(new ASTReader(
CI.getPreprocessor(), CI.getASTContext(), CI.getPCHContainerReader(),
CI.getFrontendOpts().ModuleFileExtensions,
Sysroot.empty() ? "" : Sysroot.c_str(),
/*DisableValidation*/ false,
/*AllowPCHWithCompilerErrors*/ false,
Expand Down Expand Up @@ -564,6 +569,20 @@ namespace {
}
return false;
}

/// Indicates that a particular module file extension has been read.
void readModuleFileExtension(
const ModuleFileExtensionMetadata &Metadata) override {
Out.indent(2) << "Module file extension '"
<< Metadata.BlockName << "' " << Metadata.MajorVersion
<< "." << Metadata.MinorVersion;
if (!Metadata.UserInfo.empty()) {
Out << ": ";
Out.write_escaped(Metadata.UserInfo);
}

Out << "\n";
}
#undef DUMP_BOOLEAN
};
}
Expand All @@ -583,7 +602,8 @@ void DumpModuleInfoAction::ExecuteAction() {
DumpModuleInfoListener Listener(Out);
ASTReader::readASTFileControlBlock(
getCurrentFile(), getCompilerInstance().getFileManager(),
getCompilerInstance().getPCHContainerReader(), Listener);
getCompilerInstance().getPCHContainerReader(),
/*FindModuleFileExtensions=*/true, Listener);
}

//===----------------------------------------------------------------------===//
Expand Down
121 changes: 121 additions & 0 deletions clang/lib/Frontend/TestModuleFileExtension.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
//===-- TestModuleFileExtension.cpp - Module Extension Tester -------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#include "TestModuleFileExtension.h"
#include "clang/Frontend/FrontendDiagnostic.h"
#include "clang/Serialization/ASTReader.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/Bitcode/BitstreamWriter.h"
#include "llvm/Support/raw_ostream.h"
using namespace clang;
using namespace clang::serialization;

TestModuleFileExtension::Writer::~Writer() { }

void TestModuleFileExtension::Writer::writeExtensionContents(
llvm::BitstreamWriter &Stream) {
using namespace llvm;

// Write an abbreviation for this record.
BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev();
Abv->Add(BitCodeAbbrevOp(FIRST_EXTENSION_RECORD_ID));
Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of characters
Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // message
auto Abbrev = Stream.EmitAbbrev(Abv);

// Write a message into the extension block.
SmallString<64> Message;
{
auto Ext = static_cast<TestModuleFileExtension *>(getExtension());
raw_svector_ostream OS(Message);
OS << "Hello from " << Ext->BlockName << " v" << Ext->MajorVersion << "."
<< Ext->MinorVersion;
}
SmallVector<uint64_t, 4> Record;
Record.push_back(FIRST_EXTENSION_RECORD_ID);
Record.push_back(Message.size());
Stream.EmitRecordWithBlob(Abbrev, Record, Message);
}

TestModuleFileExtension::Reader::Reader(ModuleFileExtension *Ext,
const llvm::BitstreamCursor &InStream)
: ModuleFileExtensionReader(Ext), Stream(InStream)
{
// Read the extension block.
SmallVector<uint64_t, 4> Record;
while (true) {
llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
switch (Entry.Kind) {
case llvm::BitstreamEntry::SubBlock:
case llvm::BitstreamEntry::EndBlock:
case llvm::BitstreamEntry::Error:
return;

case llvm::BitstreamEntry::Record:
break;
}

Record.clear();
StringRef Blob;
unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob);
switch (RecCode) {
case FIRST_EXTENSION_RECORD_ID: {
StringRef Message = Blob.substr(0, Record[0]);
fprintf(stderr, "Read extension block message: %s\n",
Message.str().c_str());
break;
}
}
}
}

TestModuleFileExtension::Reader::~Reader() { }

TestModuleFileExtension::~TestModuleFileExtension() { }

ModuleFileExtensionMetadata
TestModuleFileExtension::getExtensionMetadata() const {
return { BlockName, MajorVersion, MinorVersion, UserInfo };
}

llvm::hash_code TestModuleFileExtension::hashExtension(
llvm::hash_code Code) const {
if (Hashed) {
Code = llvm::hash_combine(Code, BlockName);
Code = llvm::hash_combine(Code, MajorVersion);
Code = llvm::hash_combine(Code, MinorVersion);
Code = llvm::hash_combine(Code, UserInfo);
}

return Code;
}

std::unique_ptr<ModuleFileExtensionWriter>
TestModuleFileExtension::createExtensionWriter(ASTWriter &) {
return std::unique_ptr<ModuleFileExtensionWriter>(new Writer(this));
}

std::unique_ptr<ModuleFileExtensionReader>
TestModuleFileExtension::createExtensionReader(
const ModuleFileExtensionMetadata &Metadata,
ASTReader &Reader, serialization::ModuleFile &Mod,
const llvm::BitstreamCursor &Stream)
{
assert(Metadata.BlockName == BlockName && "Wrong block name");
if (std::make_pair(Metadata.MajorVersion, Metadata.MinorVersion) !=
std::make_pair(MajorVersion, MinorVersion)) {
Reader.getDiags().Report(Mod.ImportLoc,
diag::err_test_module_file_extension_version)
<< BlockName << Metadata.MajorVersion << Metadata.MinorVersion
<< MajorVersion << MinorVersion;
return nullptr;
}

return std::unique_ptr<ModuleFileExtensionReader>(
new TestModuleFileExtension::Reader(this, Stream));
}
71 changes: 71 additions & 0 deletions clang/lib/Frontend/TestModuleFileExtension.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
//===-- TestModuleFileExtension.h - Module Extension Tester -----*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#ifndef LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H
#define LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H

#include "clang/Serialization/ModuleFileExtension.h"
#include "clang/Basic/LLVM.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Bitcode/BitstreamReader.h"
#include <string>

namespace clang {

/// A module file extension used for testing purposes.
class TestModuleFileExtension : public ModuleFileExtension {
std::string BlockName;
unsigned MajorVersion;
unsigned MinorVersion;
bool Hashed;
std::string UserInfo;

class Writer : public ModuleFileExtensionWriter {
public:
Writer(ModuleFileExtension *Ext) : ModuleFileExtensionWriter(Ext) { }
~Writer() override;

void writeExtensionContents(llvm::BitstreamWriter &Stream) override;
};

class Reader : public ModuleFileExtensionReader {
llvm::BitstreamCursor Stream;

public:
~Reader() override;

Reader(ModuleFileExtension *Ext, const llvm::BitstreamCursor &InStream);
};

public:
TestModuleFileExtension(StringRef BlockName,
unsigned MajorVersion,
unsigned MinorVersion,
bool Hashed,
StringRef UserInfo)
: BlockName(BlockName),
MajorVersion(MajorVersion), MinorVersion(MinorVersion),
Hashed(Hashed), UserInfo(UserInfo) { }
~TestModuleFileExtension() override;

ModuleFileExtensionMetadata getExtensionMetadata() const override;

llvm::hash_code hashExtension(llvm::hash_code Code) const override;

std::unique_ptr<ModuleFileExtensionWriter>
createExtensionWriter(ASTWriter &Writer) override;

std::unique_ptr<ModuleFileExtensionReader>
createExtensionReader(const ModuleFileExtensionMetadata &Metadata,
ASTReader &Reader, serialization::ModuleFile &Mod,
const llvm::BitstreamCursor &Stream) override;
};

} // end namespace clang

#endif // LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H
230 changes: 189 additions & 41 deletions clang/lib/Serialization/ASTReader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,12 @@ bool ChainedASTReaderListener::visitInputFile(StringRef Filename,
return Continue;
}

void ChainedASTReaderListener::readModuleFileExtension(
const ModuleFileExtensionMetadata &Metadata) {
First->readModuleFileExtension(Metadata);
Second->readModuleFileExtension(Metadata);
}

//===----------------------------------------------------------------------===//
// PCH validator implementation
//===----------------------------------------------------------------------===//
Expand Down Expand Up @@ -3423,6 +3429,36 @@ static void updateModuleTimestamp(ModuleFile &MF) {
OS << "Timestamp file\n";
}

/// \brief Given a cursor at the start of an AST file, scan ahead and drop the
/// cursor into the start of the given block ID, returning false on success and
/// true on failure.
static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) {
while (1) {
llvm::BitstreamEntry Entry = Cursor.advance();
switch (Entry.Kind) {
case llvm::BitstreamEntry::Error:
case llvm::BitstreamEntry::EndBlock:
return true;

case llvm::BitstreamEntry::Record:
// Ignore top-level records.
Cursor.skipRecord(Entry.ID);
break;

case llvm::BitstreamEntry::SubBlock:
if (Entry.ID == BlockID) {
if (Cursor.EnterSubBlock(BlockID))
return true;
// Found it!
return false;
}

if (Cursor.SkipBlock())
return true;
}
}
}

ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
ModuleKind Type,
SourceLocation ImportLoc,
Expand Down Expand Up @@ -3480,6 +3516,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
if (ASTReadResult Result = ReadASTBlock(F, ClientLoadCapabilities))
return Result;

// Read the extension blocks.
while (!SkipCursorToBlock(F.Stream, EXTENSION_BLOCK_ID)) {
if (ASTReadResult Result = ReadExtensionBlock(F))
return Result;
}

// Once read, set the ModuleFile bit base offset and update the size in
// bits of all files we've seen.
F.GlobalBitOffset = TotalModulesSizeInBits;
Expand Down Expand Up @@ -3738,14 +3780,13 @@ ASTReader::ReadASTCore(StringRef FileName,

// This is used for compatibility with older PCH formats.
bool HaveReadControlBlock = false;

while (1) {
llvm::BitstreamEntry Entry = Stream.advance();

switch (Entry.Kind) {
case llvm::BitstreamEntry::Error:
case llvm::BitstreamEntry::EndBlock:
case llvm::BitstreamEntry::Record:
case llvm::BitstreamEntry::EndBlock:
Error("invalid record at top-level of AST file");
return Failure;

Expand Down Expand Up @@ -3800,6 +3841,79 @@ ASTReader::ReadASTCore(StringRef FileName,
break;
}
}

return Success;
}

/// Parse a record and blob containing module file extension metadata.
static bool parseModuleFileExtensionMetadata(
const SmallVectorImpl<uint64_t> &Record,
StringRef Blob,
ModuleFileExtensionMetadata &Metadata) {
if (Record.size() < 4) return true;

Metadata.MajorVersion = Record[0];
Metadata.MinorVersion = Record[1];

unsigned BlockNameLen = Record[2];
unsigned UserInfoLen = Record[3];

if (BlockNameLen + UserInfoLen > Blob.size()) return true;

Metadata.BlockName = std::string(Blob.data(), Blob.data() + BlockNameLen);
Metadata.UserInfo = std::string(Blob.data() + BlockNameLen,
Blob.data() + BlockNameLen + UserInfoLen);
return false;
}

ASTReader::ASTReadResult ASTReader::ReadExtensionBlock(ModuleFile &F) {
BitstreamCursor &Stream = F.Stream;

RecordData Record;
while (true) {
llvm::BitstreamEntry Entry = Stream.advance();
switch (Entry.Kind) {
case llvm::BitstreamEntry::SubBlock:
if (Stream.SkipBlock())
return Failure;

continue;

case llvm::BitstreamEntry::EndBlock:
return Success;

case llvm::BitstreamEntry::Error:
return HadErrors;

case llvm::BitstreamEntry::Record:
break;
}

Record.clear();
StringRef Blob;
unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob);
switch (RecCode) {
case EXTENSION_METADATA: {
ModuleFileExtensionMetadata Metadata;
if (parseModuleFileExtensionMetadata(Record, Blob, Metadata))
return Failure;

// Find a module file extension with this block name.
auto Known = ModuleFileExtensions.find(Metadata.BlockName);
if (Known == ModuleFileExtensions.end()) break;

// Form a reader.
if (auto Reader = Known->second->createExtensionReader(Metadata, *this,
F, Stream)) {
F.ExtensionReaders.push_back(std::move(Reader));
}

break;
}
}
}

return Success;
}

void ASTReader::InitializeContext() {
Expand Down Expand Up @@ -3940,36 +4054,6 @@ void ASTReader::finalizeForWriting() {
// Nothing to do for now.
}

/// \brief Given a cursor at the start of an AST file, scan ahead and drop the
/// cursor into the start of the given block ID, returning false on success and
/// true on failure.
static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) {
while (1) {
llvm::BitstreamEntry Entry = Cursor.advance();
switch (Entry.Kind) {
case llvm::BitstreamEntry::Error:
case llvm::BitstreamEntry::EndBlock:
return true;

case llvm::BitstreamEntry::Record:
// Ignore top-level records.
Cursor.skipRecord(Entry.ID);
break;

case llvm::BitstreamEntry::SubBlock:
if (Entry.ID == BlockID) {
if (Cursor.EnterSubBlock(BlockID))
return true;
// Found it!
return false;
}

if (Cursor.SkipBlock())
return true;
}
}
}

/// \brief Reads and return the signature record from \p StreamFile's control
/// block, or else returns 0.
static ASTFileSignature readASTFileSignature(llvm::BitstreamReader &StreamFile){
Expand Down Expand Up @@ -4097,6 +4181,7 @@ namespace {
bool ASTReader::readASTFileControlBlock(
StringRef Filename, FileManager &FileMgr,
const PCHContainerReader &PCHContainerRdr,
bool FindModuleFileExtensions,
ASTReaderListener &Listener) {
// Open the AST file.
// FIXME: This allows use of the VFS; we do not allow use of the
Expand Down Expand Up @@ -4126,7 +4211,8 @@ bool ASTReader::readASTFileControlBlock(

RecordData Record;
std::string ModuleDir;
while (1) {
bool DoneWithControlBlock = false;
while (!DoneWithControlBlock) {
llvm::BitstreamEntry Entry = Stream.advance();

switch (Entry.Kind) {
Expand Down Expand Up @@ -4159,7 +4245,8 @@ bool ASTReader::readASTFileControlBlock(
}

case llvm::BitstreamEntry::EndBlock:
return false;
DoneWithControlBlock = true;
break;

case llvm::BitstreamEntry::Error:
return true;
Expand All @@ -4168,6 +4255,8 @@ bool ASTReader::readASTFileControlBlock(
break;
}

if (DoneWithControlBlock) break;

Record.clear();
StringRef Blob;
unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob);
Expand Down Expand Up @@ -4251,6 +4340,50 @@ bool ASTReader::readASTFileControlBlock(
break;
}
}

// Look for module file extension blocks, if requested.
if (FindModuleFileExtensions) {
while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) {
bool DoneWithExtensionBlock = false;
while (!DoneWithExtensionBlock) {
llvm::BitstreamEntry Entry = Stream.advance();

switch (Entry.Kind) {
case llvm::BitstreamEntry::SubBlock:
if (Stream.SkipBlock())
return true;

continue;

case llvm::BitstreamEntry::EndBlock:
DoneWithExtensionBlock = true;
continue;

case llvm::BitstreamEntry::Error:
return true;

case llvm::BitstreamEntry::Record:
break;
}

Record.clear();
StringRef Blob;
unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob);
switch (RecCode) {
case EXTENSION_METADATA: {
ModuleFileExtensionMetadata Metadata;
if (parseModuleFileExtensionMetadata(Record, Blob, Metadata))
return true;

Listener.readModuleFileExtension(Metadata);
break;
}
}
}
}
}

return false;
}

bool ASTReader::isAcceptableASTFile(
Expand All @@ -4261,6 +4394,7 @@ bool ASTReader::isAcceptableASTFile(
SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts,
ExistingModuleCachePath, FileMgr);
return !readASTFileControlBlock(Filename, FileMgr, PCHContainerRdr,
/*FindModuleFileExtensions=*/false,
validator);
}

Expand Down Expand Up @@ -8483,13 +8617,15 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) {
}
}

ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context,
const PCHContainerReader &PCHContainerRdr,
StringRef isysroot, bool DisableValidation,
bool AllowASTWithCompilerErrors,
bool AllowConfigurationMismatch, bool ValidateSystemInputs,
bool UseGlobalIndex,
std::unique_ptr<llvm::Timer> ReadTimer)
ASTReader::ASTReader(
Preprocessor &PP, ASTContext &Context,
const PCHContainerReader &PCHContainerRdr,
ArrayRef<IntrusiveRefCntPtr<ModuleFileExtension>> Extensions,
StringRef isysroot, bool DisableValidation,
bool AllowASTWithCompilerErrors,
bool AllowConfigurationMismatch, bool ValidateSystemInputs,
bool UseGlobalIndex,
std::unique_ptr<llvm::Timer> ReadTimer)
: Listener(new PCHValidator(PP, *this)), DeserializationListener(nullptr),
OwnsDeserializationListener(false), SourceMgr(PP.getSourceManager()),
FileMgr(PP.getFileManager()), PCHContainerRdr(PCHContainerRdr),
Expand All @@ -8513,6 +8649,18 @@ ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context,
TotalModulesSizeInBits(0), NumCurrentElementsDeserializing(0),
PassingDeclsToConsumer(false), ReadingKind(Read_None) {
SourceMgr.setExternalSLocEntrySource(this);

for (const auto &Ext : Extensions) {
auto BlockName = Ext->getExtensionMetadata().BlockName;
auto Known = ModuleFileExtensions.find(BlockName);
if (Known != ModuleFileExtensions.end()) {
Diags.Report(diag::warn_duplicate_module_file_extension)
<< BlockName;
continue;
}

ModuleFileExtensions.insert({BlockName, Ext});
}
}

ASTReader::~ASTReader() {
Expand Down
58 changes: 54 additions & 4 deletions clang/lib/Serialization/ASTWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//

#include "clang/Serialization/ASTWriter.h"
#include "clang/Serialization/ModuleFileExtension.h"
#include "ASTCommon.h"
#include "ASTReaderInternals.h"
#include "MultiOnDiskHashTable.h"
Expand Down Expand Up @@ -1094,7 +1095,11 @@ void ASTWriter::WriteBlockInfoBlock() {
RECORD(PPD_MACRO_EXPANSION);
RECORD(PPD_MACRO_DEFINITION);
RECORD(PPD_INCLUSION_DIRECTIVE);


// Decls and Types block.
BLOCK(EXTENSION_BLOCK);
RECORD(EXTENSION_METADATA);

#undef RECORD
#undef BLOCK
Stream.ExitBlock();
Expand Down Expand Up @@ -3876,6 +3881,40 @@ void ASTWriter::WriteOptimizePragmaOptions(Sema &SemaRef) {
Stream.EmitRecord(OPTIMIZE_PRAGMA_OPTIONS, Record);
}

void ASTWriter::WriteModuleFileExtension(ModuleFileExtensionWriter &Writer) {
// Enter the extension block.
Stream.EnterSubblock(EXTENSION_BLOCK_ID, 4);

// Emit the metadata record abbreviation.
llvm::BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev();
Abv->Add(llvm::BitCodeAbbrevOp(EXTENSION_METADATA));
Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6));
Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob));
unsigned Abbrev = Stream.EmitAbbrev(Abv);

// Emit the metadata record.
RecordData Record;
auto Metadata = Writer.getExtension()->getExtensionMetadata();
Record.push_back(EXTENSION_METADATA);
Record.push_back(Metadata.MajorVersion);
Record.push_back(Metadata.MinorVersion);
Record.push_back(Metadata.BlockName.size());
Record.push_back(Metadata.UserInfo.size());
SmallString<64> Buffer;
Buffer += Metadata.BlockName;
Buffer += Metadata.UserInfo;
Stream.EmitRecordWithBlob(Abbrev, Record, Buffer);

// Emit the contents of the extension block.
Writer.writeExtensionContents(Stream);

// Exit the extension block.
Stream.ExitBlock();
}

//===----------------------------------------------------------------------===//
// General Serialization Routines
//===----------------------------------------------------------------------===//
Expand Down Expand Up @@ -3979,7 +4018,10 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) {
SelectorOffsets[ID - FirstSelectorID] = Offset;
}

ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps)
ASTWriter::ASTWriter(
llvm::BitstreamWriter &Stream,
ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>> Extensions,
bool IncludeTimestamps)
: Stream(Stream), Context(nullptr), PP(nullptr), Chain(nullptr),
WritingModule(nullptr), IncludeTimestamps(IncludeTimestamps),
WritingAST(false), DoneWritingDeclsAndTypes(false),
Expand All @@ -3999,7 +4041,12 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps)
DeclVarAbbrev(0), DeclFieldAbbrev(0), DeclEnumAbbrev(0),
DeclObjCIvarAbbrev(0), DeclCXXMethodAbbrev(0), DeclRefExprAbbrev(0),
CharacterLiteralAbbrev(0), IntegerLiteralAbbrev(0),
ExprImplicitCastAbbrev(0) {}
ExprImplicitCastAbbrev(0) {
for (const auto &Ext : Extensions) {
if (auto Writer = Ext->createExtensionWriter(*this))
ModuleFileExtensionWriters.push_back(std::move(Writer));
}
}

ASTWriter::~ASTWriter() {
llvm::DeleteContainerSeconds(FileDeclIDs);
Expand Down Expand Up @@ -4396,7 +4443,6 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
WriteCXXCtorInitializersOffsets();
WriteFileDeclIDsMap();
WriteSourceManagerBlock(Context.getSourceManager(), PP);

WriteComments();
WritePreprocessor(PP, isModule);
WriteHeaderSearch(PP.getHeaderSearchInfo());
Expand Down Expand Up @@ -4526,6 +4572,10 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
Stream.EmitRecord(STATISTICS, Record);
Stream.ExitBlock();

// Write the module file extension blocks.
for (const auto &ExtWriter : ModuleFileExtensionWriters)
WriteModuleFileExtension(*ExtWriter);

return Signature;
}

Expand Down
1 change: 1 addition & 0 deletions clang/lib/Serialization/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ add_clang_library(clangSerialization
GeneratePCH.cpp
GlobalModuleIndex.cpp
Module.cpp
ModuleFileExtension.cpp
ModuleManager.cpp

ADDITIONAL_HEADERS
Expand Down
12 changes: 7 additions & 5 deletions clang/lib/Serialization/GeneratePCH.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@

using namespace clang;

PCHGenerator::PCHGenerator(const Preprocessor &PP, StringRef OutputFile,
clang::Module *Module, StringRef isysroot,
std::shared_ptr<PCHBuffer> Buffer,
bool AllowASTWithErrors, bool IncludeTimestamps)
PCHGenerator::PCHGenerator(
const Preprocessor &PP, StringRef OutputFile,
clang::Module *Module, StringRef isysroot,
std::shared_ptr<PCHBuffer> Buffer,
ArrayRef<llvm::IntrusiveRefCntPtr<ModuleFileExtension>> Extensions,
bool AllowASTWithErrors, bool IncludeTimestamps)
: PP(PP), OutputFile(OutputFile), Module(Module), isysroot(isysroot.str()),
SemaPtr(nullptr), Buffer(Buffer), Stream(Buffer->Data),
Writer(Stream, IncludeTimestamps),
Writer(Stream, Extensions, IncludeTimestamps),
AllowASTWithErrors(AllowASTWithErrors) {
Buffer->IsComplete = false;
}
Expand Down
22 changes: 22 additions & 0 deletions clang/lib/Serialization/ModuleFileExtension.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
//===-- ModuleFileExtension.cpp - Module File Extensions ------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
#include "clang/Serialization/ModuleFileExtension.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/Support/raw_ostream.h"
using namespace clang;

ModuleFileExtension::~ModuleFileExtension() { }

llvm::hash_code ModuleFileExtension::hashExtension(llvm::hash_code Code) const {
return Code;
}

ModuleFileExtensionWriter::~ModuleFileExtensionWriter() { }

ModuleFileExtensionReader::~ModuleFileExtensionReader() { }
1 change: 1 addition & 0 deletions clang/test/Modules/Inputs/ExtensionTestA.h
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
extern int ExtensionA;
4 changes: 4 additions & 0 deletions clang/test/Modules/Inputs/module.map
Original file line number Diff line number Diff line change
Expand Up @@ -375,3 +375,7 @@ module DebugSubmodules {
export *
}
}

module ExtensionTestA {
header "ExtensionTestA.h"
}
44 changes: 44 additions & 0 deletions clang/test/Modules/extensions.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Test creation of modules that include extension blocks.
// RUN: rm -rf %t
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s

// Make sure the extension blocks are actually there.
// RUN: llvm-bcanalyzer %t/ExtensionTestA.pcm | FileCheck -check-prefix=CHECK-BCANALYZER %s
// RUN: %clang_cc1 -module-file-info %t/ExtensionTestA.pcm | FileCheck -check-prefix=CHECK-INFO %s

// Make sure that the readers are able to check the metadata.
// RUN: rm -rf %t
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:3:0:user_info_for_A -ftest-module-file-extension=clang.testB:3:2:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s -verify

// Make sure that extension blocks can be part of the module hash.
// We test this in an obscure way, by making sure we don't get conflicts when
// using different "versions" of the extensions. Above, the "-verify" test
// checks that such conflicts produce errors.
// RUN: rm -rf %t
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:5:1:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:1:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:3:1:user_info_for_A -ftest-module-file-extension=clang.testB:3:2:1:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:2:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:7:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s

// Make sure we can read the message back.
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s > %t.log 2>&1
// RUN: FileCheck -check-prefix=CHECK-MESSAGE %s < %t.log

// Make sure we diagnose duplicate module file extensions.
// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -fmodules-cache-path=%t -I %S/Inputs %s > %t.log 2>&1
// RUN: FileCheck -check-prefix=CHECK-DUPLICATE %s < %t.log

#include "ExtensionTestA.h"
// expected-error@-1{{test module file extension 'clang.testA' has different version (1.5) than expected (1.3)}}
// expected-error@-2{{test module file extension 'clang.testB' has different version (2.3) than expected (3.2)}}

// CHECK-BCANALYZER: {{Block ID.*EXTENSION_BLOCK}}
// CHECK-BCANALYZER: {{100.00.*EXTENSION_METADATA}}

// CHECK-INFO: Module file extension 'clang.testA' 1.5: user_info_for_A
// CHECK-INFO: Module file extension 'clang.testB' 2.3: user_info_for_B

// CHECK-MESSAGE: Read extension block message: Hello from clang.testA v1.5
// CHECK-MESSAGE: Read extension block message: Hello from clang.testB v2.3

// CHECK-DUPLICATE: warning: duplicate module file extension block name 'clang.testA'