Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
#include <string>
#include <set>
#include <iostream>
#include <stdexcept>
#include <memory>
#include <chrono>
#include <sstream>
#include <boost/lexical_cast.hpp>
#include <boost/algorithm/string.hpp>
#include <thrift/transport/TBufferTransports.h>
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/protocol/TCompactProtocol.h>
#include "thrift/gen-cpp/test_types.h"
#include "thrift/gen-cpp/test_constants.h"
#include <capnp/message.h>
#include <capnp/serialize.h>
#include <cxxopts.hpp>
#include "protobuf/test.pb.h"
#include "capnproto/test.capnp.h"
#include "boost/record.hpp"
#include "msgpack/record.hpp"
#include "cereal/record.hpp"
#include "avro/record.hpp"
#include "flatbuffers/test_generated.h"
#include "yas/record.hpp"
#include "data.hpp"
enum class ThriftSerializationProto { Binary, Compact };
struct Args {
std::size_t iterations = 0;
std::set<std::string> serializers;
bool csv = false;
};
struct Result {
Result(std::string name, std::string version, size_t size, int64_t time)
: name(name)
, version(version)
, size(size)
, time(time)
{
}
Result(std::string name, uint64_t version, size_t size, int64_t time)
: name(name)
, version(boost::lexical_cast<std::string>(version))
, size(size)
, time(time)
{
}
std::string name;
std::string version;
size_t size = 0;
int64_t time = 0;
};
// clang-format off
const std::set<std::string> valid_serializers = {
"thrift-binary",
"thrift-compact",
"protobuf",
"boost",
"msgpack",
"cereal",
"avro",
"capnproto",
"flatbuffers",
"yas",
"yas-compact"
};
// clang-format on
void
print_results(Args args, std::vector<Result> results)
{
if (args.csv) { // print CSV header
std::cout << "serializer,version,iterations,size,time" << std::endl;
}
for (const auto& result : results) {
if (args.csv) {
std::cout << result.name << "," << result.version << "," << args.iterations << "," << result.size << ","
<< result.time << std::endl;
} else {
std::cout << "Serializer: " << result.name << std::endl;
std::cout << "Version : " << result.version << std::endl;
std::cout << "Iterations: " << args.iterations << std::endl;
std::cout << "Size : " << result.size << " bytes" << std::endl;
std::cout << "Time : " << result.time << " milliseconds" << std::endl;
std::cout << std::endl;
}
}
}
void
print_supported_serializers()
{
std::cerr << std::endl << "Supported serializers are:" << std::endl;
for (const auto& s : valid_serializers) {
std::cout << " * " << s << std::endl;
}
std::cout << std::endl;
}
Args
parse_args(int argc, char** argv)
{
cxxopts::Options args("benchmark", "Benchmark various C++ serializers");
// clang-format off
args.add_options()
("h,help", "show this help and exit")
("l,list", "show list of supported serializers")
("c,csv", "output in CSV format")
("i,iterations", "number of serialize/deserialize iterations", cxxopts::value<std::size_t>())
("s,serializers", "comma separated list of serializers to benchmark", cxxopts::value<std::string>())
;
// clang-format on
Args opts;
try {
auto parsed_opts = args.parse(argc, argv);
if (parsed_opts.count("help")) {
std::cout << args.help() << std::endl;
exit(EXIT_SUCCESS);
}
if (parsed_opts.count("list")) {
print_supported_serializers();
exit(EXIT_SUCCESS);
}
if (parsed_opts.count("iterations") == 0) {
std::cerr << "Not all required option specified! Please run with -h for available options." << std::endl;
exit(EXIT_FAILURE);
}
if (parsed_opts.count("serializers")) {
boost::split(opts.serializers, parsed_opts["serializers"].as<std::string>(), boost::is_any_of(","));
for (const auto& serializer : opts.serializers) {
auto exists = valid_serializers.find(serializer);
if (exists == valid_serializers.end()) {
std::cerr << "Serializer '" << serializer << "' is not supported by this benchmark." << std::endl;
print_supported_serializers();
exit(EXIT_FAILURE);
}
}
}
opts.iterations = parsed_opts["iterations"].as<std::size_t>();
if (parsed_opts.count("csv")) {
opts.csv = true;
}
} catch (std::exception& exc) {
std::cerr << exc.what() << std::endl;
exit(EXIT_FAILURE);
}
return opts;
}
Result
thrift_serialization_test(size_t iterations, ThriftSerializationProto proto = ThriftSerializationProto::Binary)
{
using apache::thrift::protocol::TBinaryProtocol;
using apache::thrift::protocol::TBinaryProtocolT;
using apache::thrift::protocol::TCompactProtocol;
using apache::thrift::protocol::TCompactProtocolT;
using apache::thrift::transport::TMemoryBuffer;
using namespace thrift_test;
std::shared_ptr<TMemoryBuffer> buffer1(new TMemoryBuffer());
std::shared_ptr<TMemoryBuffer> buffer2(new TMemoryBuffer());
TBinaryProtocolT<TMemoryBuffer> binary_protocol1(buffer1);
TBinaryProtocolT<TMemoryBuffer> binary_protocol2(buffer2);
TCompactProtocolT<TMemoryBuffer> compact_protocol1(buffer1);
TCompactProtocolT<TMemoryBuffer> compact_protocol2(buffer2);
Record r1;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
if (proto == ThriftSerializationProto::Binary) {
r1.write(&binary_protocol1);
} else if (proto == ThriftSerializationProto::Compact) {
r1.write(&compact_protocol1);
}
serialized = buffer1->getBufferAsString();
// check if we can deserialize back
Record r2;
buffer2->resetBuffer((uint8_t*)serialized.data(), serialized.length());
if (proto == ThriftSerializationProto::Binary) {
r2.read(&binary_protocol2);
} else if (proto == ThriftSerializationProto::Compact) {
r2.read(&compact_protocol2);
}
if (r1 != r2) {
throw std::logic_error("thrift's case: deserialization failed");
}
std::string tag;
if (proto == ThriftSerializationProto::Binary) {
tag = "thrift-binary";
} else if (proto == ThriftSerializationProto::Compact) {
tag = "thrift-compact";
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
buffer1->resetBuffer();
if (proto == ThriftSerializationProto::Binary) {
r1.write(&binary_protocol1);
} else if (proto == ThriftSerializationProto::Compact) {
r1.write(&compact_protocol1);
}
serialized = buffer1->getBufferAsString();
buffer2->resetBuffer((uint8_t*)serialized.data(), serialized.length());
if (proto == ThriftSerializationProto::Binary) {
r2.read(&binary_protocol2);
} else if (proto == ThriftSerializationProto::Compact) {
r2.read(&compact_protocol2);
}
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result(tag, PACKAGE_VERSION, serialized.size(), duration);
}
Result
protobuf_serialization_test(size_t iterations)
{
using namespace protobuf_test;
Record r1;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.add_ids(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.add_strings(kStringValue);
}
std::string serialized;
r1.SerializeToString(&serialized);
// check if we can deserialize back
Record r2;
bool ok = r2.ParseFromString(serialized);
if (!ok /*|| r2 != r1*/) {
throw std::logic_error("protobuf's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
serialized.clear();
r1.SerializeToString(&serialized);
r2.ParseFromString(serialized);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("protobuf", GOOGLE_PROTOBUF_VERSION, serialized.size(), duration);
}
Result
capnproto_serialization_test(size_t iterations)
{
using namespace capnp_test;
capnp::MallocMessageBuilder message;
Record::Builder r1 = message.getRoot<Record>();
auto ids = r1.initIds(kIntegers.size());
for (size_t i = 0; i < kIntegers.size(); i++) {
ids.set(i, kIntegers[i]);
}
auto strings = r1.initStrings(kStringsCount);
for (size_t i = 0; i < kStringsCount; i++) {
strings.set(i, kStringValue);
}
kj::ArrayPtr<const kj::ArrayPtr<const capnp::word>> serialized = message.getSegmentsForOutput();
// check if we can deserialize back
capnp::SegmentArrayMessageReader reader(serialized);
Record::Reader r2 = reader.getRoot<Record>();
if (r2.getIds().size() != kIntegers.size()) {
throw std::logic_error("capnproto's case: deserialization failed");
}
size_t size = 0;
for (auto segment : serialized) {
size += segment.asBytes().size();
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
capnp::MallocMessageBuilder message;
Record::Builder r1 = message.getRoot<Record>();
auto ids = r1.initIds(kIntegers.size());
for (size_t i = 0; i < kIntegers.size(); i++) {
ids.set(i, kIntegers[i]);
}
auto strings = r1.initStrings(kStringsCount);
for (size_t i = 0; i < kStringsCount; i++) {
strings.set(i, kStringValue);
}
serialized = message.getSegmentsForOutput();
capnp::SegmentArrayMessageReader reader(serialized);
auto r2 = reader.getRoot<Record>();
(void)r2.getIds().size();
(void)r2.getStrings().size();
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("capnproto", CAPNP_VERSION, size, duration);
}
Result
boost_serialization_test(size_t iterations)
{
using namespace boost_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
to_string(r1, serialized);
from_string(r2, serialized);
if (r1 != r2) {
throw std::logic_error("boost's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
serialized.clear();
to_string(r1, serialized);
from_string(r2, serialized);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("boost", BOOST_VERSION, serialized.size(), duration);
}
Result
msgpack_serialization_test(size_t iterations)
{
using namespace msgpack_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
msgpack::sbuffer sbuf;
msgpack::pack(sbuf, r1);
std::string serialized(sbuf.data(), sbuf.size());
msgpack::object_handle msg = msgpack::unpack(serialized.data(), serialized.size());
msgpack::object obj = msg.get();
obj.convert(r2);
if (r1 != r2) {
throw std::logic_error("msgpack's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
sbuf.clear();
msgpack::pack(sbuf, r1);
msgpack::object_handle msg = msgpack::unpack(sbuf.data(), sbuf.size());
msgpack::object obj = msg.get();
obj.convert(r2);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("msgpack", msgpack_version(), serialized.size(), duration);
}
Result
cereal_serialization_test(size_t iterations)
{
using namespace cereal_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
to_string(r1, serialized);
from_string(r2, serialized);
if (r1 != r2) {
throw std::logic_error("cereal's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
serialized.clear();
to_string(r1, serialized);
from_string(r2, serialized);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("cereal", "", serialized.size(), duration);
}
Result
avro_serialization_test(size_t iterations)
{
using namespace avro_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::unique_ptr<avro::OutputStream> out = avro::memoryOutputStream();
avro::EncoderPtr encoder = avro::binaryEncoder();
encoder->init(*out);
avro::encode(*encoder, r1);
auto serialized_size = out->byteCount();
std::unique_ptr<avro::InputStream> in = avro::memoryInputStream(*out);
avro::DecoderPtr decoder = avro::binaryDecoder();
decoder->init(*in);
avro::decode(*decoder, r2);
if (r1.ids != r2.ids || r1.strings != r2.strings || r2.ids.size() != kIntegers.size()
|| r2.strings.size() != kStringsCount) {
throw std::logic_error("avro's case: deserialization failed");
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
auto out = avro::memoryOutputStream();
auto encoder = avro::binaryEncoder();
encoder->init(*out);
avro::encode(*encoder, r1);
auto in = avro::memoryInputStream(*out);
auto decoder = avro::binaryDecoder();
decoder->init(*in);
avro::decode(*decoder, r2);
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result("avro", "", serialized_size, duration);
}
Result
flatbuffers_serialization_test(size_t iterations)
{
using namespace flatbuffers_test;
std::vector<flatbuffers::Offset<flatbuffers::String>> strings;
strings.reserve(kStringsCount);
flatbuffers::FlatBufferBuilder builder;
for (size_t i = 0; i < kStringsCount; i++) {
strings.push_back(builder.CreateString(kStringValue));
}
auto ids_vec = builder.CreateVector(kIntegers);
auto strings_vec = builder.CreateVector(strings);
auto r1 = CreateRecord(builder, ids_vec, strings_vec);
builder.Finish(r1);
auto p = reinterpret_cast<char*>(builder.GetBufferPointer());
auto sz = builder.GetSize();
std::vector<char> buf(p, p + sz);
auto r2 = GetRecord(buf.data());
if (r2->strings()->size() != kStringsCount || r2->ids()->size() != kIntegers.size()) {
throw std::logic_error("flatbuffer's case: deserialization failed");
}
auto size = builder.GetSize();
builder.ReleaseBufferPointer();
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
flatbuffers::FlatBufferBuilder builder;
strings.clear();
for (size_t i = 0; i < kStringsCount; i++) {
strings.push_back(builder.CreateString(kStringValue));
}
auto ids_vec = builder.CreateVector(kIntegers);
auto strings_vec = builder.CreateVector(strings);
auto r1 = CreateRecord(builder, ids_vec, strings_vec);
builder.Finish(r1);
auto p = reinterpret_cast<char*>(builder.GetBufferPointer());
auto sz = builder.GetSize();
std::vector<char> buf(p, p + sz);
auto r2 = GetRecord(buf.data());
(void)r2->ids()[0];
builder.ReleaseBufferPointer();
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
auto version = FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MAJOR) "." FLATBUFFERS_STRING(
FLATBUFFERS_VERSION_MINOR) "." FLATBUFFERS_STRING(FLATBUFFERS_VERSION_REVISION);
return Result("flatbuffers", version, size, duration);
}
template <std::size_t opts>
Result
yas_serialization_test(size_t iterations)
{
using namespace yas_test;
Record r1, r2;
for (size_t i = 0; i < kIntegers.size(); i++) {
r1.ids.push_back(kIntegers[i]);
}
for (size_t i = 0; i < kStringsCount; i++) {
r1.strings.push_back(kStringValue);
}
std::string serialized;
to_string<opts>(r1, serialized);
from_string<opts>(r2, serialized);
if (r1 != r2) {
throw std::logic_error("yas' case: deserialization failed");
}
std::string tag;
if (opts & yas::compacted) {
tag = "yas-compact";
} else {
tag = "yas";
}
auto start = std::chrono::high_resolution_clock::now();
for (size_t i = 0; i < iterations; i++) {
yas::mem_ostream os;
yas::binary_oarchive<yas::mem_ostream, opts> oa(os);
oa& r1;
yas::mem_istream is(os.get_intrusive_buffer());
yas::binary_iarchive<yas::mem_istream, opts> ia(is);
ia& r2;
}
auto finish = std::chrono::high_resolution_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count();
return Result(tag, YAS_VERSION_STRING, serialized.size(), duration);
}
int
main(int argc, char** argv)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
auto args = parse_args(argc, argv);
/*std::cout << "total size: " << sizeof(kIntegerValue) * kIntegersCount + kStringValue.size() * kStringsCount <<
* std::endl;*/
std::vector<Result> results;
try {
if (args.serializers.empty() || args.serializers.find("thrift-binary") != args.serializers.end()) {
results.push_back(thrift_serialization_test(args.iterations, ThriftSerializationProto::Binary));
}
if (args.serializers.empty() || args.serializers.find("thrift-compact") != args.serializers.end()) {
results.push_back(thrift_serialization_test(args.iterations, ThriftSerializationProto::Compact));
}
if (args.serializers.empty() || args.serializers.find("protobuf") != args.serializers.end()) {
results.push_back(protobuf_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("capnproto") != args.serializers.end()) {
results.push_back(capnproto_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("boost") != args.serializers.end()) {
results.push_back(boost_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("msgpack") != args.serializers.end()) {
results.push_back(msgpack_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("cereal") != args.serializers.end()) {
results.push_back(cereal_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("avro") != args.serializers.end()) {
results.push_back(avro_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("flatbuffers") != args.serializers.end()) {
results.push_back(flatbuffers_serialization_test(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("yas") != args.serializers.end()) {
results.push_back(yas_serialization_test<yas::binary | yas::no_header>(args.iterations));
}
if (args.serializers.empty() || args.serializers.find("yas-compact") != args.serializers.end()) {
results.push_back(yas_serialization_test<yas::binary | yas::no_header | yas::compacted>(args.iterations));
}
} catch (std::exception& exc) {
std::cerr << "Error: " << exc.what() << std::endl;
return EXIT_FAILURE;
}
print_results(args, results);
google::protobuf::ShutdownProtobufLibrary();
return EXIT_SUCCESS;
}