Make the clang module container format selectable from the command line.

- introduces a new cc1 option -fmodule-format=[raw,obj]
  with 'raw' being the default
- supports arbitrary module container formats that libclang is agnostic to
- adds the format to the module hash to avoid collisions
- splits the old PCHContainerOperations into PCHContainerWriter and
  a PCHContainerReader.

Thanks to Richard Smith for reviewing this patch!

llvm-svn: 242499

clang/include/clang/Basic/DiagnosticCommonKinds.td

@@ -195,6 +195,8 @@ def err_unable_to_make_temp : Error<
 
 // Modules
 def err_module_file_conflict : Error<"module '%0' found in both '%1' and '%2'">;
+def err_module_format_unhandled : Error<
+  "no handler registered for module format '%0'">;
 
 // TransformActions
 // TODO: Use a custom category name to distinguish rewriter errors.

clang/include/clang/CodeGen/ObjectFilePCHContainerOperations.h

@@ -14,30 +14,32 @@
 
 namespace clang {
 
-/// \brief A PCHContainerOperations implementation that uses LLVM to
+/// A PCHContainerWriter implementation that uses LLVM to
 /// wraps Clang modules inside a COFF, ELF, or Mach-O container.
-class ObjectFilePCHContainerOperations
-  : public PCHContainerOperations {
-  /// \brief Return an ASTConsumer that can be chained with a
+class ObjectFilePCHContainerWriter : public PCHContainerWriter {
+  StringRef getFormat() const override { return "obj"; }
+
+  /// Return an ASTConsumer that can be chained with a
   /// PCHGenerator that produces a wrapper file format
   /// that also contains full debug info for the module.
-  std::unique_ptr<ASTConsumer>
-    CreatePCHContainerGenerator(
+  std::unique_ptr<ASTConsumer> CreatePCHContainerGenerator(
       DiagnosticsEngine &Diags, const HeaderSearchOptions &HSO,
       const PreprocessorOptions &PPO, const TargetOptions &TO,
       const LangOptions &LO, const std::string &MainFileName,
       const std::string &OutputFileName, llvm::raw_pwrite_stream *OS,
       std::shared_ptr<PCHBuffer> Buffer) const override;
+};
+
+/// A PCHContainerReader implementation that uses LLVM to
+/// wraps Clang modules inside a COFF, ELF, or Mach-O container.
+class ObjectFilePCHContainerReader : public PCHContainerReader {
+  StringRef getFormat() const override { return "obj"; }
 
-  /// \brief Initialize an llvm::BitstreamReader with the serialized
+  /// Initialize an llvm::BitstreamReader with the serialized
   /// AST inside the PCH container Buffer.
   void ExtractPCH(llvm::MemoryBufferRef Buffer,
                   llvm::BitstreamReader &StreamFile) const override;
-
-
 };
-
 }
 
-
 #endif

clang/include/clang/Driver/CC1Options.td

@@ -369,6 +369,9 @@ def fmodules_local_submodule_visibility :
   Flag<["-"], "fmodules-local-submodule-visibility">,
   HelpText<"Enforce name visibility rules across submodules of the same "
            "top-level module.">;
+def fmodule_format_EQ : Joined<["-"], "fmodule-format=">,
+  HelpText<"Select the container format for clang modules and PCH. "
+           "Supported options are 'raw' and 'obj'.">;
 def fno_modules_hide_internal_linkage :
   Flag<["-"], "fno-modules-hide-internal-linkage">,
   HelpText<"Make all declarations visible to redeclaration lookup, "

clang/include/clang/Frontend/ASTUnit.h

@@ -57,6 +57,7 @@ class FileManager;
 class HeaderSearch;
 class Preprocessor;
 class PCHContainerOperations;
+class PCHContainerReader;
 class SourceManager;
 class TargetInfo;
 class ASTFrontendAction;
@@ -725,8 +726,7 @@ class ASTUnit : public ModuleLoader {
   ///
   /// \returns - The initialized ASTUnit or null if the AST failed to load.
   static std::unique_ptr<ASTUnit> LoadFromASTFile(
-      const std::string &Filename,
-      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+      const std::string &Filename, const PCHContainerReader &PCHContainerRdr,
       IntrusiveRefCntPtr<DiagnosticsEngine> Diags,
       const FileSystemOptions &FileSystemOpts, bool OnlyLocalDecls = false,
       ArrayRef<RemappedFile> RemappedFiles = None,

clang/include/clang/Frontend/CompilerInstance.h

@@ -183,7 +183,7 @@ class CompilerInstance : public ModuleLoader {
 public:
   explicit CompilerInstance(
       std::shared_ptr<PCHContainerOperations> PCHContainerOps =
-          std::make_shared<RawPCHContainerOperations>(),
+          std::make_shared<PCHContainerOperations>(),
       bool BuildingModule = false);
   ~CompilerInstance() override;
 
@@ -508,6 +508,34 @@ class CompilerInstance : public ModuleLoader {
     return ThePCHContainerOperations;
   }
 
+  /// Return the appropriate PCHContainerWriter depending on the
+  /// current CodeGenOptions.
+  const PCHContainerWriter &getPCHContainerWriter() const {
+    assert(Invocation && "cannot determine module format without invocation");
+    StringRef Format = getHeaderSearchOpts().ModuleFormat;
+    auto *Writer = ThePCHContainerOperations->getWriterOrNull(Format);
+    if (!Writer) {
+      if (Diagnostics)
+        Diagnostics->Report(diag::err_module_format_unhandled) << Format;
+      llvm::report_fatal_error("unknown module format");
+    }
+    return *Writer;
+  }
+
+  /// Return the appropriate PCHContainerReader depending on the
+  /// current CodeGenOptions.
+  const PCHContainerReader &getPCHContainerReader() const {
+    assert(Invocation && "cannot determine module format without invocation");
+    StringRef Format = getHeaderSearchOpts().ModuleFormat;
+    auto *Reader = ThePCHContainerOperations->getReaderOrNull(Format);
+    if (!Reader) {
+      if (Diagnostics)
+        Diagnostics->Report(diag::err_module_format_unhandled) << Format;
+      llvm::report_fatal_error("unknown module format");
+    }
+    return *Reader;
+  }
+
   /// }
   /// @name Code Completion
   /// {
@@ -621,7 +649,7 @@ class CompilerInstance : public ModuleLoader {
   static IntrusiveRefCntPtr<ASTReader> createPCHExternalASTSource(
       StringRef Path, StringRef Sysroot, bool DisablePCHValidation,
       bool AllowPCHWithCompilerErrors, Preprocessor &PP, ASTContext &Context,
-      const PCHContainerOperations &PCHContainerOps,
+      const PCHContainerReader &PCHContainerRdr,
       void *DeserializationListener, bool OwnDeserializationListener,
       bool Preamble, bool UseGlobalModuleIndex);
 

clang/include/clang/Frontend/PCHContainerOperations.h

@@ -11,6 +11,7 @@
 #define LLVM_CLANG_PCH_CONTAINER_OPERATIONS_H
 
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include <memory>
 
@@ -19,6 +20,8 @@ class raw_pwrite_stream;
 class BitstreamReader;
 }
 
+using llvm::StringRef;
+
 namespace clang {
 
 class ASTConsumer;
@@ -33,14 +36,16 @@ struct PCHBuffer {
   bool IsComplete;
   llvm::SmallVector<char, 0> Data;
 };
+
+/// This abstract interface provides operations for creating
+/// containers for serialized ASTs (precompiled headers and clang
+/// modules).
+class PCHContainerWriter {
+public: 
+  virtual ~PCHContainerWriter() = 0;
+  virtual StringRef getFormat() const = 0;
 
-/// \brief This abstract interface provides operations for creating
-/// and unwrapping containers for serialized ASTs (precompiled headers
-/// and clang modules).
-class PCHContainerOperations {
-public:
-  virtual ~PCHContainerOperations();
-  /// \brief Return an ASTConsumer that can be chained with a
+  /// Return an ASTConsumer that can be chained with a
   /// PCHGenerator that produces a wrapper file format containing a
   /// serialized AST bitstream.
   virtual std::unique_ptr<ASTConsumer> CreatePCHContainerGenerator(
@@ -49,28 +54,71 @@ class PCHContainerOperations {
       const LangOptions &LO, const std::string &MainFileName,
       const std::string &OutputFileName, llvm::raw_pwrite_stream *OS,
       std::shared_ptr<PCHBuffer> Buffer) const = 0;
+};
 
-  /// \brief Initialize an llvm::BitstreamReader with the serialized AST inside
+/// This abstract interface provides operations for unwrapping
+/// containers for serialized ASTs (precompiled headers and clang
+/// modules).
+class PCHContainerReader {
+public: 
+  virtual ~PCHContainerReader() = 0;
+  /// Equivalent to the format passed to -fmodule-format=
+  virtual StringRef getFormat() const = 0;
+
+  /// Initialize an llvm::BitstreamReader with the serialized AST inside
   /// the PCH container Buffer.
   virtual void ExtractPCH(llvm::MemoryBufferRef Buffer,
                           llvm::BitstreamReader &StreamFile) const = 0;
 };
 
-/// \brief Implements a raw pass-through PCH container.
-class RawPCHContainerOperations : public PCHContainerOperations {
-  /// \brief Return an ASTConsumer that can be chained with a
+/// Implements write operations for a raw pass-through PCH container.
+class RawPCHContainerWriter : public PCHContainerWriter {
+  StringRef getFormat() const override { return "raw"; }
+
+  /// Return an ASTConsumer that can be chained with a
   /// PCHGenerator that writes the module to a flat file.
   std::unique_ptr<ASTConsumer> CreatePCHContainerGenerator(
       DiagnosticsEngine &Diags, const HeaderSearchOptions &HSO,
       const PreprocessorOptions &PPO, const TargetOptions &TO,
       const LangOptions &LO, const std::string &MainFileName,
       const std::string &OutputFileName, llvm::raw_pwrite_stream *OS,
       std::shared_ptr<PCHBuffer> Buffer) const override;
+};
 
-  /// \brief Initialize an llvm::BitstreamReader with Buffer.
+/// Implements read operations for a raw pass-through PCH container.
+class RawPCHContainerReader : public PCHContainerReader {
+  StringRef getFormat() const override { return "raw"; }
+
+  /// Initialize an llvm::BitstreamReader with Buffer.
   void ExtractPCH(llvm::MemoryBufferRef Buffer,
                   llvm::BitstreamReader &StreamFile) const override;
 };
+
+/// A registry of PCHContainerWriter and -Reader objects for different formats.
+class PCHContainerOperations {
+  llvm::StringMap<std::unique_ptr<PCHContainerWriter>> Writers;
+  llvm::StringMap<std::unique_ptr<PCHContainerReader>> Readers;
+public:
+  /// Automatically registers a RawPCHContainerWriter and
+  /// RawPCHContainerReader.
+  PCHContainerOperations();
+  void registerWriter(std::unique_ptr<PCHContainerWriter> Writer) {
+    Writers[Writer->getFormat()] = std::move(Writer);
+  }
+  void registerReader(std::unique_ptr<PCHContainerReader> Reader) {
+    Readers[Reader->getFormat()] = std::move(Reader);
+  }  
+  const PCHContainerWriter *getWriterOrNull(StringRef Format) {
+    return Writers[Format].get();
+  }
+  const PCHContainerReader *getReaderOrNull(StringRef Format) {
+    return Readers[Format].get();
+  }
+  const PCHContainerReader &getRawReader() {
+    return *getReaderOrNull("raw");
+  }
+};
+
 }
 
 #endif

clang/include/clang/Frontend/Utils.h

@@ -45,7 +45,7 @@ class HeaderSearch;
 class HeaderSearchOptions;
 class IdentifierTable;
 class LangOptions;
-class PCHContainerOperations;
+class PCHContainerReader;
 class Preprocessor;
 class PreprocessorOptions;
 class PreprocessorOutputOptions;
@@ -63,7 +63,7 @@ void ApplyHeaderSearchOptions(HeaderSearch &HS,
 /// InitializePreprocessor - Initialize the preprocessor getting it and the
 /// environment ready to process a single file.
 void InitializePreprocessor(Preprocessor &PP, const PreprocessorOptions &PPOpts,
-                            const PCHContainerOperations &PCHContainerOps,
+                            const PCHContainerReader &PCHContainerRdr,
                             const FrontendOptions &FEOpts);
 
 /// DoPrintPreprocessedInput - Implement -E mode.

clang/include/clang/Lex/HeaderSearchOptions.h

@@ -92,6 +92,9 @@ class HeaderSearchOptions : public RefCountedBase<HeaderSearchOptions> {
   /// \brief The directory used for a user build.
   std::string ModuleUserBuildPath;
 
+  /// The module/pch container format.
+  std::string ModuleFormat;
+
   /// \brief Whether we should disable the use of the hash string within the
   /// module cache.
   ///
@@ -167,16 +170,14 @@ class HeaderSearchOptions : public RefCountedBase<HeaderSearchOptions> {
 
 public:
   HeaderSearchOptions(StringRef _Sysroot = "/")
-    : Sysroot(_Sysroot), DisableModuleHash(0), ImplicitModuleMaps(0),
-      ModuleMapFileHomeIsCwd(0),
-      ModuleCachePruneInterval(7*24*60*60),
-      ModuleCachePruneAfter(31*24*60*60),
-      BuildSessionTimestamp(0),
-      UseBuiltinIncludes(true),
-      UseStandardSystemIncludes(true), UseStandardCXXIncludes(true),
-      UseLibcxx(false), Verbose(false),
-      ModulesValidateOncePerBuildSession(false),
-      ModulesValidateSystemHeaders(false) {}
+      : Sysroot(_Sysroot), ModuleFormat("raw"), DisableModuleHash(0),
+        ImplicitModuleMaps(0), ModuleMapFileHomeIsCwd(0),
+        ModuleCachePruneInterval(7 * 24 * 60 * 60),
+        ModuleCachePruneAfter(31 * 24 * 60 * 60), BuildSessionTimestamp(0),
+        UseBuiltinIncludes(true), UseStandardSystemIncludes(true),
+        UseStandardCXXIncludes(true), UseLibcxx(false), Verbose(false),
+        ModulesValidateOncePerBuildSession(false),
+        ModulesValidateSystemHeaders(false) {}
 
   /// AddPath - Add the \p Path path to the specified \p Group list.
   void AddPath(StringRef Path, frontend::IncludeDirGroup Group,

clang/include/clang/Serialization/ASTReader.h

@@ -362,7 +362,7 @@ class ASTReader
 
   SourceManager &SourceMgr;
   FileManager &FileMgr;
-  const PCHContainerOperations &PCHContainerOps;
+  const PCHContainerReader &PCHContainerRdr;
   DiagnosticsEngine &Diags;
 
   /// \brief The semantic analysis object that will be processing the
@@ -1289,7 +1289,7 @@ class ASTReader
   /// \param ReadTimer If non-null, a timer used to track the time spent
   /// deserializing.
   ASTReader(Preprocessor &PP, ASTContext &Context,
-            const PCHContainerOperations &PCHContainerOps,
+            const PCHContainerReader &PCHContainerRdr,
             StringRef isysroot = "", bool DisableValidation = false,
             bool AllowASTWithCompilerErrors = false,
             bool AllowConfigurationMismatch = false,
@@ -1458,21 +1458,21 @@ class ASTReader
   /// the AST file, without actually loading the AST file.
   static std::string
   getOriginalSourceFile(const std::string &ASTFileName, FileManager &FileMgr,
-                        const PCHContainerOperations &PCHContainerOps,
+                        const PCHContainerReader &PCHContainerRdr,
                         DiagnosticsEngine &Diags);
 
   /// \brief Read the control block for the named AST file.
   ///
   /// \returns true if an error occurred, false otherwise.
   static bool
   readASTFileControlBlock(StringRef Filename, FileManager &FileMgr,
-                          const PCHContainerOperations &PCHContainerOps,
+                          const PCHContainerReader &PCHContainerRdr,
                           ASTReaderListener &Listener);
 
   /// \brief Determine whether the given AST file is acceptable to load into a
   /// translation unit with the given language and target options.
   static bool isAcceptableASTFile(StringRef Filename, FileManager &FileMgr,
-                                  const PCHContainerOperations &PCHContainerOps,
+                                  const PCHContainerReader &PCHContainerRdr,
                                   const LangOptions &LangOpts,
                                   const TargetOptions &TargetOpts,
                                   const PreprocessorOptions &PPOpts,

clang/include/clang/Serialization/GlobalModuleIndex.h

@@ -198,10 +198,9 @@ class GlobalModuleIndex {
   /// \param Path The path to the directory containing module files, into
   /// which the global index will be written.
   static ErrorCode writeIndex(FileManager &FileMgr,
-                              const PCHContainerOperations &PCHContainerOps,
+                              const PCHContainerReader &PCHContainerRdr,
                               StringRef Path);
 };
-
 }
 
 #endif

clang/include/clang/Serialization/ModuleManager.h

@@ -24,7 +24,7 @@ namespace clang {
 
 class GlobalModuleIndex;
 class ModuleMap;
-class PCHContainerOperations;
+class PCHContainerReader;
 
 namespace serialization {
 
@@ -52,7 +52,7 @@ class ModuleManager {
   FileManager &FileMgr;
 
   /// \brief Knows how to unwrap module containers.
-  const PCHContainerOperations &PCHContainerOps;
+  const PCHContainerReader &PCHContainerRdr;
 
   /// \brief A lookup of in-memory (virtual file) buffers
   llvm::DenseMap<const FileEntry *, std::unique_ptr<llvm::MemoryBuffer>>
@@ -118,9 +118,9 @@ class ModuleManager {
   typedef std::pair<uint32_t, StringRef> ModuleOffset;
 
   explicit ModuleManager(FileManager &FileMgr,
-                         const PCHContainerOperations &PCHContainerOps);
+                         const PCHContainerReader &PCHContainerRdr);
   ~ModuleManager();
-  
+
   /// \brief Forward iterator to traverse all loaded modules.  This is reverse
   /// source-order.
   ModuleIterator begin() { return Chain.begin(); }

clang/include/clang/Tooling/Refactoring.h

@@ -40,7 +40,7 @@ class RefactoringTool : public ClangTool {
   RefactoringTool(const CompilationDatabase &Compilations,
                   ArrayRef<std::string> SourcePaths,
                   std::shared_ptr<PCHContainerOperations> PCHContainerOps =
-                      std::make_shared<RawPCHContainerOperations>());
+                      std::make_shared<PCHContainerOperations>());
 
   /// \brief Returns the set of replacements to which replacements should
   /// be added during the run of the tool.