[Remarks] Add two serialization modes for remarks: separate and stand…

…alone

The default mode is separate, where the metadata is serialized
separately from the remarks.

Another mode is the standalone mode, where the metadata is serialized
before the remarks, on the same stream.

llvm-svn: 367328

llvm/docs/Remarks.rst

@@ -147,6 +147,26 @@ Other tools that support remarks:
     .. option:: -opt-remarks-format=<format>
     .. option:: -opt-remarks-with-hotness
 
+Serialization modes
+===================
+
+There are two modes available for serializing remarks:
+
+``Separate``
+
+    In this mode, the remarks and the metadata are serialized separately. The
+    client is responsible for parsing the metadata first, then use the metadata
+    to correctly parse the remarks.
+
+``Standalone``
+
+    In this mode, the remarks and the metadata are serialized to the same
+    stream. The metadata will always come before the remarks.
+
+    The compiler does not support emitting standalone remarks. This mode is
+    more suited for post-processing tools like linkers, that can merge the
+    remarks for one whole project.
+
 .. _yamlremarks:
 
 YAML remarks

llvm/include/llvm/Remarks/RemarkSerializer.h

@@ -14,24 +14,38 @@
 #define LLVM_REMARKS_REMARK_SERIALIZER_H
 
 #include "llvm/Remarks/Remark.h"
+#include "llvm/Remarks/RemarkFormat.h"
 #include "llvm/Remarks/RemarkStringTable.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 namespace remarks {
 
+enum class SerializerMode {
+  Separate,  // A mode where the metadata is serialized separately from the
+             // remarks. Typically, this is used when the remarks need to be
+             // streamed to a side file and the metadata is embedded into the
+             // final result of the compilation.
+  Standalone // A mode where everything can be retrieved in the same
+             // file/buffer. Typically, this is used for storing remarks for
+             // later use.
+};
+
 struct MetaSerializer;
 
 /// This is the base class for a remark serializer.
 /// It includes support for using a string table while emitting.
 struct RemarkSerializer {
   /// The open raw_ostream that the remark diagnostics are emitted to.
   raw_ostream &OS;
+  /// The serialization mode.
+  SerializerMode Mode;
   /// The string table containing all the unique strings used in the output.
   /// The table can be serialized to be consumed after the compilation.
   Optional<StringTable> StrTab;
 
-  RemarkSerializer(raw_ostream &OS) : OS(OS), StrTab() {}
+  RemarkSerializer(raw_ostream &OS, SerializerMode Mode)
+      : OS(OS), Mode(Mode), StrTab() {}
 
   /// This is just an interface.
   virtual ~RemarkSerializer() = default;
@@ -57,12 +71,12 @@ struct MetaSerializer {
 
 /// Create a remark serializer.
 Expected<std::unique_ptr<RemarkSerializer>>
-createRemarkSerializer(Format RemarksFormat, raw_ostream &OS);
+createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, raw_ostream &OS);
 
 /// Create a remark serializer that uses a pre-filled string table.
 Expected<std::unique_ptr<RemarkSerializer>>
-createRemarkSerializer(Format RemarksFormat, raw_ostream &OS,
-                       remarks::StringTable StrTab);
+createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
+                       raw_ostream &OS, remarks::StringTable StrTab);
 
 } // end namespace remarks
 } // end namespace llvm

llvm/include/llvm/Remarks/YAMLRemarkSerializer.h

@@ -34,12 +34,15 @@ struct YAMLRemarkSerializer : public RemarkSerializer {
   /// The YAML streamer.
   yaml::Output YAMLOutput;
 
-  YAMLRemarkSerializer(raw_ostream &OS);
+  YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode);
 
   void emit(const Remark &Remark) override;
   std::unique_ptr<MetaSerializer>
   metaSerializer(raw_ostream &OS,
                  Optional<StringRef> ExternalFilename = None) override;
+
+protected:
+  bool DidEmitMeta = false;
 };
 
 struct YAMLMetaSerializer : public MetaSerializer {
@@ -55,12 +58,14 @@ struct YAMLMetaSerializer : public MetaSerializer {
 /// like the regular YAML remark but instead of string entries it's using
 /// numbers that map to an index in the string table.
 struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
-  YAMLStrTabRemarkSerializer(raw_ostream &OS) : YAMLRemarkSerializer(OS) {
+  YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode)
+      : YAMLRemarkSerializer(OS, Mode) {
     // Having a string table set up enables the serializer to use it.
     StrTab.emplace();
   }
-  YAMLStrTabRemarkSerializer(raw_ostream &OS, StringTable StrTabIn)
-      : YAMLRemarkSerializer(OS) {
+  YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode,
+                             StringTable StrTabIn)
+      : YAMLRemarkSerializer(OS, Mode) {
     StrTab = std::move(StrTabIn);
   }
   std::unique_ptr<MetaSerializer>

llvm/lib/IR/RemarkStreamer.cpp

@@ -136,7 +136,7 @@ llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename,
     return make_error<RemarkSetupFormatError>(std::move(E));
 
   Expected<std::unique_ptr<remarks::RemarkSerializer>> RemarkSerializer =
-      remarks::createRemarkSerializer(*Format, RemarksFile->os());
+      remarks::createRemarkSerializer(*Format, remarks::SerializerMode::Separate, RemarksFile->os());
   if (Error E = RemarkSerializer.takeError())
     return make_error<RemarkSetupFormatError>(std::move(E));
 

llvm/lib/Remarks/RemarkSerializer.cpp

@@ -17,22 +17,23 @@ using namespace llvm;
 using namespace llvm::remarks;
 
 Expected<std::unique_ptr<RemarkSerializer>>
-remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS) {
+remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
+                                raw_ostream &OS) {
   switch (RemarksFormat) {
   case Format::Unknown:
     return createStringError(std::errc::invalid_argument,
                              "Unknown remark serializer format.");
   case Format::YAML:
-    return llvm::make_unique<YAMLRemarkSerializer>(OS);
+    return llvm::make_unique<YAMLRemarkSerializer>(OS, Mode);
   case Format::YAMLStrTab:
-    return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS);
+    return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS, Mode);
   }
   llvm_unreachable("Unknown remarks::Format enum");
 }
 
 Expected<std::unique_ptr<RemarkSerializer>>
-remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS,
-                                remarks::StringTable StrTab) {
+remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
+                                raw_ostream &OS, remarks::StringTable StrTab) {
   switch (RemarksFormat) {
   case Format::Unknown:
     return createStringError(std::errc::invalid_argument,
@@ -42,7 +43,8 @@ remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS,
                              "Unable to use a string table with the yaml "
                              "format. Use 'yaml-strtab' instead.");
   case Format::YAMLStrTab:
-    return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS, std::move(StrTab));
+    return llvm::make_unique<YAMLStrTabRemarkSerializer>(OS, Mode,
+                                                         std::move(StrTab));
   }
   llvm_unreachable("Unknown remarks::Format enum");
 }

llvm/lib/Remarks/YAMLRemarkSerializer.cpp

@@ -149,10 +149,19 @@ template <> struct MappingTraits<Argument> {
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(Argument)
 
-YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS)
-    : RemarkSerializer(OS), YAMLOutput(OS, reinterpret_cast<void *>(this)) {}
+YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode)
+    : RemarkSerializer(OS, Mode), YAMLOutput(OS, reinterpret_cast<void *>(this)) {}
 
 void YAMLRemarkSerializer::emit(const Remark &Remark) {
+  // In standalone mode, emit the metadata first and set DidEmitMeta to avoid
+  // emitting it again.
+  if (Mode == SerializerMode::Standalone) {
+    std::unique_ptr<MetaSerializer> MetaSerializer =
+        metaSerializer(OS, /*ExternalFilename=*/None);
+    MetaSerializer->emit();
+    DidEmitMeta = true;
+  }
+
   // Again, YAMLTraits expect a non-const object for inputting, but we're not
   // using that here.
   auto R = const_cast<remarks::Remark *>(&Remark);

llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp

@@ -21,32 +21,49 @@
 
 using namespace llvm;
 
-static void check(const remarks::Remark &R, StringRef ExpectedR,
-                  StringRef ExpectedMeta, bool UseStrTab = false,
+static void check(remarks::SerializerMode Mode, const remarks::Remark &R,
+                  StringRef ExpectedR, Optional<StringRef> ExpectedMeta,
+                  bool UseStrTab = false,
                   Optional<remarks::StringTable> StrTab = None) {
   std::string Buf;
   raw_string_ostream OS(Buf);
   Expected<std::unique_ptr<remarks::RemarkSerializer>> MaybeS = [&] {
     if (UseStrTab) {
       if (StrTab)
-        return createRemarkSerializer(remarks::Format::YAMLStrTab, OS,
+        return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS,
                                       std::move(*StrTab));
       else
-        return createRemarkSerializer(remarks::Format::YAMLStrTab, OS);
+        return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS);
     } else
-      return createRemarkSerializer(remarks::Format::YAML, OS);
+      return createRemarkSerializer(remarks::Format::YAML, Mode, OS);
   }();
   EXPECT_FALSE(errorToBool(MaybeS.takeError()));
   std::unique_ptr<remarks::RemarkSerializer> S = std::move(*MaybeS);
 
   S->emit(R);
   EXPECT_EQ(OS.str(), ExpectedR);
 
-  Buf.clear();
-  std::unique_ptr<remarks::MetaSerializer> MS =
-      S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH));
-  MS->emit();
-  EXPECT_EQ(OS.str(), ExpectedMeta);
+  if (ExpectedMeta) {
+    Buf.clear();
+    std::unique_ptr<remarks::MetaSerializer> MS =
+        S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH));
+    MS->emit();
+    EXPECT_EQ(OS.str(), *ExpectedMeta);
+  }
+}
+
+static void check(const remarks::Remark &R, StringRef ExpectedR,
+                  StringRef ExpectedMeta, bool UseStrTab = false,
+                  Optional<remarks::StringTable> StrTab = None) {
+  return check(remarks::SerializerMode::Separate, R, ExpectedR, ExpectedMeta,
+               UseStrTab, std::move(StrTab));
+}
+
+static void checkStandalone(const remarks::Remark &R, StringRef ExpectedR,
+                            Optional<remarks::StringTable> StrTab = None) {
+  bool UseStrTab = StrTab.hasValue();
+  return check(remarks::SerializerMode::Standalone, R, ExpectedR,
+               /*ExpectedMeta=*/None, UseStrTab, std::move(StrTab));
 }
 
 TEST(YAMLRemarks, SerializerRemark) {
@@ -83,6 +100,40 @@ TEST(YAMLRemarks, SerializerRemark) {
                   38));
 }
 
+TEST(YAMLRemarks, SerializerRemarkStandalone) {
+  remarks::Remark R;
+  R.RemarkType = remarks::Type::Missed;
+  R.PassName = "pass";
+  R.RemarkName = "name";
+  R.FunctionName = "func";
+  R.Loc = remarks::RemarkLocation{"path", 3, 4};
+  R.Hotness = 5;
+  R.Args.emplace_back();
+  R.Args.back().Key = "key";
+  R.Args.back().Val = "value";
+  R.Args.emplace_back();
+  R.Args.back().Key = "keydebug";
+  R.Args.back().Val = "valuedebug";
+  R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
+  checkStandalone(
+      R,
+      StringRef("REMARKS\0"
+                "\0\0\0\0\0\0\0\0"
+                "\0\0\0\0\0\0\0\0"
+                "--- !Missed\n"
+                "Pass:            pass\n"
+                "Name:            name\n"
+                "DebugLoc:        { File: path, Line: 3, Column: 4 }\n"
+                "Function:        func\n"
+                "Hotness:         5\n"
+                "Args:\n"
+                "  - key:             value\n"
+                "  - keydebug:        valuedebug\n"
+                "    DebugLoc:        { File: argpath, Line: 6, Column: 7 }\n"
+                "...\n",
+                301));
+}
+
 TEST(YAMLRemarks, SerializerRemarkStrTab) {
   remarks::Remark R;
   R.RemarkType = remarks::Type::Missed;
@@ -156,3 +207,42 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTab) {
         /*UseStrTab=*/true,
         remarks::StringTable(remarks::ParsedStringTable(StrTab)));
 }
+
+TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandalone) {
+  StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
+  remarks::ParsedStringTable ParsedStrTab(StrTab);
+  remarks::StringTable PreFilledStrTab(ParsedStrTab);
+  remarks::Remark R;
+  R.RemarkType = remarks::Type::Missed;
+  R.PassName = "pass";
+  R.RemarkName = "name";
+  R.FunctionName = "func";
+  R.Loc = remarks::RemarkLocation{"path", 3, 4};
+  R.Hotness = 5;
+  R.Args.emplace_back();
+  R.Args.back().Key = "key";
+  R.Args.back().Val = "value";
+  R.Args.emplace_back();
+  R.Args.back().Key = "keydebug";
+  R.Args.back().Val = "valuedebug";
+  R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
+  checkStandalone(
+      R,
+      StringRef("REMARKS\0"
+                "\0\0\0\0\0\0\0\0"
+                "\x2d\0\0\0\0\0\0\0"
+                "pass\0name\0func\0path\0value\0valuedebug\0argpath\0"
+                "--- !Missed\n"
+                "Pass:            0\n"
+                "Name:            1\n"
+                "DebugLoc:        { File: 3, Line: 3, Column: 4 }\n"
+                "Function:        2\n"
+                "Hotness:         5\n"
+                "Args:\n"
+                "  - key:             4\n"
+                "  - keydebug:        5\n"
+                "    DebugLoc:        { File: 6, Line: 6, Column: 7 }\n"
+                "...\n",
+                315),
+      std::move(PreFilledStrTab));
+}