diff --git a/llvm/docs/Remarks.rst b/llvm/docs/Remarks.rst index c573eb2dec5db..f4e5d8f45e6af 100644 --- a/llvm/docs/Remarks.rst +++ b/llvm/docs/Remarks.rst @@ -147,6 +147,26 @@ Other tools that support remarks: .. option:: -opt-remarks-format= .. option:: -opt-remarks-with-hotness +Serialization modes +=================== + +There are two modes available for serializing remarks: + +``Separate`` + + In this mode, the remarks and the metadata are serialized separately. The + client is responsible for parsing the metadata first, then use the metadata + to correctly parse the remarks. + +``Standalone`` + + In this mode, the remarks and the metadata are serialized to the same + stream. The metadata will always come before the remarks. + + The compiler does not support emitting standalone remarks. This mode is + more suited for post-processing tools like linkers, that can merge the + remarks for one whole project. + .. _yamlremarks: YAML remarks diff --git a/llvm/include/llvm/Remarks/RemarkSerializer.h b/llvm/include/llvm/Remarks/RemarkSerializer.h index a8e083f54587d..cea2afb2d378b 100644 --- a/llvm/include/llvm/Remarks/RemarkSerializer.h +++ b/llvm/include/llvm/Remarks/RemarkSerializer.h @@ -14,12 +14,23 @@ #define LLVM_REMARKS_REMARK_SERIALIZER_H #include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkFormat.h" #include "llvm/Remarks/RemarkStringTable.h" #include "llvm/Support/raw_ostream.h" namespace llvm { namespace remarks { +enum class SerializerMode { + Separate, // A mode where the metadata is serialized separately from the + // remarks. Typically, this is used when the remarks need to be + // streamed to a side file and the metadata is embedded into the + // final result of the compilation. + Standalone // A mode where everything can be retrieved in the same + // file/buffer. Typically, this is used for storing remarks for + // later use. +}; + struct MetaSerializer; /// This is the base class for a remark serializer. @@ -27,11 +38,14 @@ struct MetaSerializer; struct RemarkSerializer { /// The open raw_ostream that the remark diagnostics are emitted to. raw_ostream &OS; + /// The serialization mode. + SerializerMode Mode; /// The string table containing all the unique strings used in the output. /// The table can be serialized to be consumed after the compilation. Optional StrTab; - RemarkSerializer(raw_ostream &OS) : OS(OS), StrTab() {} + RemarkSerializer(raw_ostream &OS, SerializerMode Mode) + : OS(OS), Mode(Mode), StrTab() {} /// This is just an interface. virtual ~RemarkSerializer() = default; @@ -57,12 +71,12 @@ struct MetaSerializer { /// Create a remark serializer. Expected> -createRemarkSerializer(Format RemarksFormat, raw_ostream &OS); +createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, raw_ostream &OS); /// Create a remark serializer that uses a pre-filled string table. Expected> -createRemarkSerializer(Format RemarksFormat, raw_ostream &OS, - remarks::StringTable StrTab); +createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS, remarks::StringTable StrTab); } // end namespace remarks } // end namespace llvm diff --git a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h index 5d68bb4c6eb86..f57ae4c7c5f93 100644 --- a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h +++ b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h @@ -34,12 +34,15 @@ struct YAMLRemarkSerializer : public RemarkSerializer { /// The YAML streamer. yaml::Output YAMLOutput; - YAMLRemarkSerializer(raw_ostream &OS); + YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode); void emit(const Remark &Remark) override; std::unique_ptr metaSerializer(raw_ostream &OS, Optional ExternalFilename = None) override; + +protected: + bool DidEmitMeta = false; }; struct YAMLMetaSerializer : public MetaSerializer { @@ -55,12 +58,14 @@ struct YAMLMetaSerializer : public MetaSerializer { /// like the regular YAML remark but instead of string entries it's using /// numbers that map to an index in the string table. struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer { - YAMLStrTabRemarkSerializer(raw_ostream &OS) : YAMLRemarkSerializer(OS) { + YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode) + : YAMLRemarkSerializer(OS, Mode) { // Having a string table set up enables the serializer to use it. StrTab.emplace(); } - YAMLStrTabRemarkSerializer(raw_ostream &OS, StringTable StrTabIn) - : YAMLRemarkSerializer(OS) { + YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode, + StringTable StrTabIn) + : YAMLRemarkSerializer(OS, Mode) { StrTab = std::move(StrTabIn); } std::unique_ptr diff --git a/llvm/lib/IR/RemarkStreamer.cpp b/llvm/lib/IR/RemarkStreamer.cpp index 450453fc680b6..4bbbfe839c95d 100644 --- a/llvm/lib/IR/RemarkStreamer.cpp +++ b/llvm/lib/IR/RemarkStreamer.cpp @@ -136,7 +136,7 @@ llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, return make_error(std::move(E)); Expected> RemarkSerializer = - remarks::createRemarkSerializer(*Format, RemarksFile->os()); + remarks::createRemarkSerializer(*Format, remarks::SerializerMode::Separate, RemarksFile->os()); if (Error E = RemarkSerializer.takeError()) return make_error(std::move(E)); diff --git a/llvm/lib/Remarks/RemarkSerializer.cpp b/llvm/lib/Remarks/RemarkSerializer.cpp index aa68b497ef230..73cec4fbf9ba7 100644 --- a/llvm/lib/Remarks/RemarkSerializer.cpp +++ b/llvm/lib/Remarks/RemarkSerializer.cpp @@ -17,22 +17,23 @@ using namespace llvm; using namespace llvm::remarks; Expected> -remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS) { +remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS) { switch (RemarksFormat) { case Format::Unknown: return createStringError(std::errc::invalid_argument, "Unknown remark serializer format."); case Format::YAML: - return llvm::make_unique(OS); + return llvm::make_unique(OS, Mode); case Format::YAMLStrTab: - return llvm::make_unique(OS); + return llvm::make_unique(OS, Mode); } llvm_unreachable("Unknown remarks::Format enum"); } Expected> -remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS, - remarks::StringTable StrTab) { +remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, + raw_ostream &OS, remarks::StringTable StrTab) { switch (RemarksFormat) { case Format::Unknown: return createStringError(std::errc::invalid_argument, @@ -42,7 +43,8 @@ remarks::createRemarkSerializer(Format RemarksFormat, raw_ostream &OS, "Unable to use a string table with the yaml " "format. Use 'yaml-strtab' instead."); case Format::YAMLStrTab: - return llvm::make_unique(OS, std::move(StrTab)); + return llvm::make_unique(OS, Mode, + std::move(StrTab)); } llvm_unreachable("Unknown remarks::Format enum"); } diff --git a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp index 725ac15d7eaad..f8ed740725954 100644 --- a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp +++ b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp @@ -149,10 +149,19 @@ template <> struct MappingTraits { LLVM_YAML_IS_SEQUENCE_VECTOR(Argument) -YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS) - : RemarkSerializer(OS), YAMLOutput(OS, reinterpret_cast(this)) {} +YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode) + : RemarkSerializer(OS, Mode), YAMLOutput(OS, reinterpret_cast(this)) {} void YAMLRemarkSerializer::emit(const Remark &Remark) { + // In standalone mode, emit the metadata first and set DidEmitMeta to avoid + // emitting it again. + if (Mode == SerializerMode::Standalone) { + std::unique_ptr MetaSerializer = + metaSerializer(OS, /*ExternalFilename=*/None); + MetaSerializer->emit(); + DidEmitMeta = true; + } + // Again, YAMLTraits expect a non-const object for inputting, but we're not // using that here. auto R = const_cast(&Remark); diff --git a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp index 811c1b2a455a0..761a46f0ba81e 100644 --- a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp +++ b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp @@ -21,20 +21,21 @@ using namespace llvm; -static void check(const remarks::Remark &R, StringRef ExpectedR, - StringRef ExpectedMeta, bool UseStrTab = false, +static void check(remarks::SerializerMode Mode, const remarks::Remark &R, + StringRef ExpectedR, Optional ExpectedMeta, + bool UseStrTab = false, Optional StrTab = None) { std::string Buf; raw_string_ostream OS(Buf); Expected> MaybeS = [&] { if (UseStrTab) { if (StrTab) - return createRemarkSerializer(remarks::Format::YAMLStrTab, OS, + return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS, std::move(*StrTab)); else - return createRemarkSerializer(remarks::Format::YAMLStrTab, OS); + return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS); } else - return createRemarkSerializer(remarks::Format::YAML, OS); + return createRemarkSerializer(remarks::Format::YAML, Mode, OS); }(); EXPECT_FALSE(errorToBool(MaybeS.takeError())); std::unique_ptr S = std::move(*MaybeS); @@ -42,11 +43,27 @@ static void check(const remarks::Remark &R, StringRef ExpectedR, S->emit(R); EXPECT_EQ(OS.str(), ExpectedR); - Buf.clear(); - std::unique_ptr MS = - S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH)); - MS->emit(); - EXPECT_EQ(OS.str(), ExpectedMeta); + if (ExpectedMeta) { + Buf.clear(); + std::unique_ptr MS = + S->metaSerializer(OS, StringRef(EXTERNALFILETESTPATH)); + MS->emit(); + EXPECT_EQ(OS.str(), *ExpectedMeta); + } +} + +static void check(const remarks::Remark &R, StringRef ExpectedR, + StringRef ExpectedMeta, bool UseStrTab = false, + Optional StrTab = None) { + return check(remarks::SerializerMode::Separate, R, ExpectedR, ExpectedMeta, + UseStrTab, std::move(StrTab)); +} + +static void checkStandalone(const remarks::Remark &R, StringRef ExpectedR, + Optional StrTab = None) { + bool UseStrTab = StrTab.hasValue(); + return check(remarks::SerializerMode::Standalone, R, ExpectedR, + /*ExpectedMeta=*/None, UseStrTab, std::move(StrTab)); } TEST(YAMLRemarks, SerializerRemark) { @@ -83,6 +100,40 @@ TEST(YAMLRemarks, SerializerRemark) { 38)); } +TEST(YAMLRemarks, SerializerRemarkStandalone) { + remarks::Remark R; + R.RemarkType = remarks::Type::Missed; + R.PassName = "pass"; + R.RemarkName = "name"; + R.FunctionName = "func"; + R.Loc = remarks::RemarkLocation{"path", 3, 4}; + R.Hotness = 5; + R.Args.emplace_back(); + R.Args.back().Key = "key"; + R.Args.back().Val = "value"; + R.Args.emplace_back(); + R.Args.back().Key = "keydebug"; + R.Args.back().Val = "valuedebug"; + R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7}; + checkStandalone( + R, + StringRef("REMARKS\0" + "\0\0\0\0\0\0\0\0" + "\0\0\0\0\0\0\0\0" + "--- !Missed\n" + "Pass: pass\n" + "Name: name\n" + "DebugLoc: { File: path, Line: 3, Column: 4 }\n" + "Function: func\n" + "Hotness: 5\n" + "Args:\n" + " - key: value\n" + " - keydebug: valuedebug\n" + " DebugLoc: { File: argpath, Line: 6, Column: 7 }\n" + "...\n", + 301)); +} + TEST(YAMLRemarks, SerializerRemarkStrTab) { remarks::Remark R; R.RemarkType = remarks::Type::Missed; @@ -156,3 +207,42 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTab) { /*UseStrTab=*/true, remarks::StringTable(remarks::ParsedStringTable(StrTab))); } + +TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandalone) { + StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45); + remarks::ParsedStringTable ParsedStrTab(StrTab); + remarks::StringTable PreFilledStrTab(ParsedStrTab); + remarks::Remark R; + R.RemarkType = remarks::Type::Missed; + R.PassName = "pass"; + R.RemarkName = "name"; + R.FunctionName = "func"; + R.Loc = remarks::RemarkLocation{"path", 3, 4}; + R.Hotness = 5; + R.Args.emplace_back(); + R.Args.back().Key = "key"; + R.Args.back().Val = "value"; + R.Args.emplace_back(); + R.Args.back().Key = "keydebug"; + R.Args.back().Val = "valuedebug"; + R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7}; + checkStandalone( + R, + StringRef("REMARKS\0" + "\0\0\0\0\0\0\0\0" + "\x2d\0\0\0\0\0\0\0" + "pass\0name\0func\0path\0value\0valuedebug\0argpath\0" + "--- !Missed\n" + "Pass: 0\n" + "Name: 1\n" + "DebugLoc: { File: 3, Line: 3, Column: 4 }\n" + "Function: 2\n" + "Hotness: 5\n" + "Args:\n" + " - key: 4\n" + " - keydebug: 5\n" + " DebugLoc: { File: 6, Line: 6, Column: 7 }\n" + "...\n", + 315), + std::move(PreFilledStrTab)); +}