Skip to content
Permalink
master
Go to file
 
 
Cannot retrieve contributors at this time
706 lines (546 sloc) 21.1 KB
#include <string>
#include <set>
#include <iostream>
#include <stdexcept>
#include <memory>
#include <chrono>
#include <sstream>
#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string.hpp>
#include <thrift/transport/TBufferTransports.h>
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/protocol/TCompactProtocol.h>
#include "thrift/gen-cpp/test_types.h"
#include "thrift/gen-cpp/test_constants.h"
#include <capnp/message.h>
#include <capnp/serialize.h>
#include <cxxopts.hpp>
#include "protobuf/test.pb.h"
#include "capnproto/test.capnp.h"
#include "boost/record.hpp"
#include "msgpack/record.hpp"
#include "cereal/record.hpp"
#include "avro/record.hpp"
#include "flatbuffers/test_generated.h"
#include "yas/record.hpp"
#include "data.hpp"
enum class ThriftSerializationProto { Binary, Compact };
struct Args {
std::size_t iterations = 0;
std::set<std::string> serializers;
bool csv = false;
};
struct Result {
Result(std::string name, std::string version, size_t size, int64_t time)
: name(name)
, version(version)
, size(size)
, time(time)
{
}
Result(std::string name, uint64_t version, size_t size, int64_t time)
: name(name)
, version(boost::lexical_cast<std::string>(version))
, size(size)
, time(time)
{
}
std::string name;
std::string version;
size_t size = 0;
int64_t time = 0;
};
// clang-format off
const std::set<std::string> valid_serializers = {
"thrift-binary",
"thrift-compact",
"protobuf",
"boost",
"msgpack",
"cereal",
"avro",
"capnproto",
"flatbuffers",
"yas",
"yas-compact"
};
// clang-format on
void
print_results(Args args, std::vector<Result> results)
{
if (args.csv) { // print CSV header
std::cout << "serializer,version,iterations,size,time" << std::endl;
}
for (const auto& result : results) {
if (args.csv) {
std::cout << result.name << "," << result.version << "," << args.iterations << "," << result.size << ","
<< result.time << std::endl;
} else {
std::cout << "Serializer: " << result.name << std::endl;
std::cout << "Version : " << result.version << std::endl;
std::cout << "Iterations: " << args.iterations << std::endl;
std::cout << "Size : " << result.size << " bytes" << std::endl;
std::cout << "Time : " << result.time << " milliseconds" << std::endl;
std::cout << std::endl;
}
}
}
void
print_supported_serializers()
{
std::cerr << std::endl << "Supported serializers are:" << std::endl;
for (const auto& s : valid_serializers) {
std::cout << " * " << s << std::endl;
}
std::cout << std::endl;
}
Args
parse_args(int argc, char** argv)
{
cxxopts::Options args("benchmark", "Benchmark various C++ serializers");
// clang-format off
args.add_options()
("h,help", "show this help and exit")
("l,list", "show list of supported serializers")
("c,csv", "output in CSV format")
("i,iterations", "number of serialize/deserialize iterations", cxxopts::value<std::size_t>())
("s,serializers", "comma separated list of serializers to benchmark", cxxopts::value<std::string>())
;
// clang-format on
Args opts;
try {
auto parsed_opts = args.parse(argc, argv);
if (parsed_opts.count("help")) {
std::cout << args.help() << std::endl;
exit(EXIT_SUCCESS);
}
if (parsed_opts.count("list")) {
print_supported_serializers();
exit(EXIT_SUCCESS);
}
if (parsed_opts.count("iterations") == 0) {
std::cerr << "Not all required option specified! Please run with -h for available options." << std::endl;
exit(EXIT_FAILURE);
}
if (parsed_opts.count("serializers")) {
boost::split(opts.serializers, parsed_opts["serializers"].as<std::string>(), boost::is_any_of(","));
for (const auto& serializer : opts.serializers) {
auto exists = valid_serializers.find(serializer);
if (exists == valid_serializers.end()) {
std::cerr << "Serializer '" << serializer << "' is not supported by this benchmark." << std::endl;
print_supported_serializers();
exit(EXIT_FAILURE);
}
}
}
opts.iterations = parsed_opts["iterations"].as<std::size_t>();
if (parsed_opts.count("csv")) {
opts.csv = true;
}
} catch (std::exception& exc) {
std::cerr << exc.what() << std::endl;
exit(EXIT_FAILURE);
}
return opts;
}
Result
thrift_serialization_test(size_t iterations, ThriftSerializationProto proto = ThriftSerializationProto::Binary)
{
using apache::thrift::protocol::TBinaryProtocol;
using apache::thrift::protocol::TBinaryProtocolT;
using apache::thrift::protocol::TCompactProtocol;
using apache::thrift::protocol::TCompactProtocolT;
using apache::thrift::transport::TMemoryBuffer;
using namespace thrift_test;
std::shared_ptr<TMemoryBuffer> buffer1(new TMemoryBuffer());
std::shared_ptr<TMemoryBuffer> buffer2(new TMemoryBuffer());
TBinaryProtocolT<TMemoryBuffer> binary_protocol1(buffer1);
TBinaryProtocolT<TMemoryBuffer> binary_protocol2(buffer2);
TCompactProtocolT<TMemoryBuffer> compact_protocol1(buffer1);
TCompactProtocolT<TMemoryBuffer> compact_protocol2(buffer2);
Record r1;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
if (proto == ThriftSerializationProto::Binary) {
r1.write(&binary_protocol1);
} else if (proto == ThriftSerializationProto::Compact) {
r1.write(&compact_protocol1);
}
serialized = buffer1->getBufferAsString();
// check if we can deserialize back
Record r2;
buffer2->resetBuffer((uint8_t*)serialized.data(), serialized.length());
if (proto == ThriftSerializationProto::Binary) {
r2.read(&binary_protocol2);
} else if (proto == ThriftSerializationProto::Compact) {
r2.read(&compact_protocol2);
}
if (r1 != r2) {
throw std::logic_error("thrift's case: deserialization failed");
}
std::string tag;
if (proto == ThriftSerializationProto::Binary) {
tag = "thrift-binary";
} else if (proto == ThriftSerializationProto::Compact) {
tag = "thrift-compact";
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
buffer1->resetBuffer();
if (proto == ThriftSerializationProto::Binary) {
r1.write(&binary_protocol1);
} else if (proto == ThriftSerializationProto::Compact) {
r1.write(&compact_protocol1);
}
serialized = buffer1->getBufferAsString();
buffer2->resetBuffer((uint8_t*)serialized.data(), serialized.length());
if (proto == ThriftSerializationProto::Binary) {
r2.read(&binary_protocol2);
} else if (proto == ThriftSerializationProto::Compact) {
r2.read(&compact_protocol2);
}
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result(tag, PACKAGE_VERSION, serialized.size(), duration);
}
Result
protobuf_serialization_test(size_t iterations)
{
using namespace protobuf_test;
Record r1;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.add_ids(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.add_strings(kStringValue);
}
std::string serialized;
r1.SerializeToString(&serialized);
// check if we can deserialize back
Record r2;
bool ok = r2.ParseFromString(serialized);
if (!ok /*|| r2 != r1*/) {
throw std::logic_error("protobuf's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
serialized.clear();
r1.SerializeToString(&serialized);
r2.ParseFromString(serialized);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("protobuf", GOOGLE_PROTOBUF_VERSION, serialized.size(), duration);
}
Result
capnproto_serialization_test(size_t iterations)
{
using namespace capnp_test;
capnp::MallocMessageBuilder message;
Record::Builder r1 = message.getRoot<Record>();
auto ids = r1.initIds(kIntegers.size());
for (size_t i = 0; i < kIntegers.size(); i++) {
ids.set(i, kIntegers[i]);
}
auto strings = r1.initStrings(kStringsCount);
for (size_t i = 0; i < kStringsCount; i++) {
strings.set(i, kStringValue);
}
kj::ArrayPtr<const kj::ArrayPtr<const capnp::word>> serialized = message.getSegmentsForOutput();
// check if we can deserialize back
capnp::SegmentArrayMessageReader reader(serialized);
Record::Reader r2 = reader.getRoot<Record>();
if (r2.getIds().size() != kIntegers.size()) {
throw std::logic_error("capnproto's case: deserialization failed");
}
size_t size = 0;
for (auto segment : serialized) {
size += segment.asBytes().size();
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
capnp::MallocMessageBuilder message;
Record::Builder r1 = message.getRoot<Record>();
auto ids = r1.initIds(kIntegers.size());
for (size_t i = 0; i < kIntegers.size(); i++) {
ids.set(i, kIntegers[i]);
}
auto strings = r1.initStrings(kStringsCount);
for (size_t i = 0; i < kStringsCount; i++) {
strings.set(i, kStringValue);
}
serialized = message.getSegmentsForOutput();
capnp::SegmentArrayMessageReader reader(serialized);
auto r2 = reader.getRoot<Record>();
(void)r2.getIds().size();
(void)r2.getStrings().size();
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("capnproto", CAPNP_VERSION, size, duration);
}
Result
boost_serialization_test(size_t iterations)
{
using namespace boost_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
to_string(r1, serialized);
from_string(r2, serialized);
if (r1 != r2) {
throw std::logic_error("boost's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
serialized.clear();
to_string(r1, serialized);
from_string(r2, serialized);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("boost", BOOST_VERSION, serialized.size(), duration);
}
Result
msgpack_serialization_test(size_t iterations)
{
using namespace msgpack_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
msgpack::sbuffer sbuf;
msgpack::pack(sbuf, r1);
std::string serialized(sbuf.data(), sbuf.size());
msgpack::object_handle msg = msgpack::unpack(serialized.data(), serialized.size());
msgpack::object obj = msg.get();
obj.convert(r2);
if (r1 != r2) {
throw std::logic_error("msgpack's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
sbuf.clear();
msgpack::pack(sbuf, r1);
msgpack::object_handle msg = msgpack::unpack(sbuf.data(), sbuf.size());
msgpack::object obj = msg.get();
obj.convert(r2);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("msgpack", msgpack_version(), serialized.size(), duration);
}
Result
cereal_serialization_test(size_t iterations)
{
using namespace cereal_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
to_string(r1, serialized);
from_string(r2, serialized);
if (r1 != r2) {
throw std::logic_error("cereal's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
serialized.clear();
to_string(r1, serialized);
from_string(r2, serialized);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("cereal", "", serialized.size(), duration);
}
Result
avro_serialization_test(size_t iterations)
{
using namespace avro_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::unique_ptr<avro::OutputStream> out = avro::memoryOutputStream();
avro::EncoderPtr encoder = avro::binaryEncoder();
encoder->init(*out);
avro::encode(*encoder, r1);
auto serialized_size = out->byteCount();
std::unique_ptr<avro::InputStream> in = avro::memoryInputStream(*out);
avro::DecoderPtr decoder = avro::binaryDecoder();
decoder->init(*in);
avro::decode(*decoder, r2);
if (r1.ids != r2.ids || r1.strings != r2.strings || r2.ids.size() != kIntegers.size()
|| r2.strings.size() != kStringsCount) {
throw std::logic_error("avro's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
auto out = avro::memoryOutputStream();
auto encoder = avro::binaryEncoder();
encoder->init(*out);
avro::encode(*encoder, r1);
auto in = avro::memoryInputStream(*out);
auto decoder = avro::binaryDecoder();
decoder->init(*in);
avro::decode(*decoder, r2);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("avro", "", serialized_size, duration);
}
Result
flatbuffers_serialization_test(size_t iterations)
{
using namespace flatbuffers_test;
std::vector<flatbuffers::Offset<flatbuffers::String>> strings;
strings.reserve(kStringsCount);
flatbuffers::FlatBufferBuilder builder;
for (size_t i = 0; i < kStringsCount; i++) {
strings.push_back(builder.CreateString(kStringValue));
}
auto ids_vec = builder.CreateVector(kIntegers);
auto strings_vec = builder.CreateVector(strings);
auto r1 = CreateRecord(builder, ids_vec, strings_vec);
builder.Finish(r1);
auto p = reinterpret_cast<char*>(builder.GetBufferPointer());
auto sz = builder.GetSize();
std::vector<char> buf(p, p + sz);
auto r2 = GetRecord(buf.data());
if (r2->strings()->size() != kStringsCount || r2->ids()->size() != kIntegers.size()) {
throw std::logic_error("flatbuffer's case: deserialization failed");
}
auto size = builder.GetSize();
builder.ReleaseBufferPointer();
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
flatbuffers::FlatBufferBuilder builder;
strings.clear();
for (size_t i = 0; i < kStringsCount; i++) {
strings.push_back(builder.CreateString(kStringValue));
}
auto ids_vec = builder.CreateVector(kIntegers);
auto strings_vec = builder.CreateVector(strings);
auto r1 = CreateRecord(builder, ids_vec, strings_vec);
builder.Finish(r1);
auto p = reinterpret_cast<char*>(builder.GetBufferPointer());
auto sz = builder.GetSize();
std::vector<char> buf(p, p + sz);
auto r2 = GetRecord(buf.data());
(void)r2->ids()[0];
builder.ReleaseBufferPointer();
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
auto version = FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MAJOR) "." FLATBUFFERS_STRING(
FLATBUFFERS_VERSION_MINOR) "." FLATBUFFERS_STRING(FLATBUFFERS_VERSION_REVISION);
return Result("flatbuffers", version, size, duration);
}
template <std::size_t opts>
Result
yas_serialization_test(size_t iterations)
{
using namespace yas_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
to_string<opts>(r1, serialized);
from_string<opts>(r2, serialized);
if (r1 != r2) {
throw std::logic_error("yas' case: deserialization failed");
}
std::string tag;
if (opts & yas::compacted) {
tag = "yas-compact";
} else {
tag = "yas";
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
yas::mem_ostream os;
yas::binary_oarchive<yas::mem_ostream, opts> oa(os);
oa& r1;
yas::mem_istream is(os.get_intrusive_buffer());
yas::binary_iarchive<yas::mem_istream, opts> ia(is);
ia& r2;
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result(tag, YAS_VERSION_STRING, serialized.size(), duration);
}
int
main(int argc, char** argv)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
auto args = parse_args(argc, argv);
/*std::cout << "total size: " << sizeof(kIntegerValue) * kIntegersCount + kStringValue.size() * kStringsCount <<
* std::endl;*/
std::vector<Result> results;
try {
if (args.serializers.empty() || args.serializers.find("thrift-binary") != args.serializers.end()) {
results.push_back(thrift_serialization_test(args.iterations, ThriftSerializationProto::Binary));
}
if (args.serializers.empty() || args.serializers.find("thrift-compact") != args.serializers.end()) {
results.push_back(thrift_serialization_test(args.iterations, ThriftSerializationProto::Compact));
}
if (args.serializers.empty() || args.serializers.find("protobuf") != args.serializers.end()) {
results.push_back(protobuf_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("capnproto") != args.serializers.end()) {
results.push_back(capnproto_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("boost") != args.serializers.end()) {
results.push_back(boost_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("msgpack") != args.serializers.end()) {
results.push_back(msgpack_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("cereal") != args.serializers.end()) {
results.push_back(cereal_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("avro") != args.serializers.end()) {
results.push_back(avro_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("flatbuffers") != args.serializers.end()) {
results.push_back(flatbuffers_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("yas") != args.serializers.end()) {
results.push_back(yas_serialization_test<yas::binary | yas::no_header>(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("yas-compact") != args.serializers.end()) {
results.push_back(yas_serialization_test<yas::binary | yas::no_header | yas::compacted>(args.iterations));
}
} catch (std::exception& exc) {
std::cerr << "Error: " << exc.what() << std::endl;
return EXIT_FAILURE;
}
print_results(args, results);
google::protobuf::ShutdownProtobufLibrary();
return EXIT_SUCCESS;
}