Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
cpp-serializers/benchmark.cpp
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
706 lines (546 sloc)
21.1 KB
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#include <string> | |
#include <set> | |
#include <iostream> | |
#include <stdexcept> | |
#include <memory> | |
#include <chrono> | |
#include <sstream> | |
#include <boost/lexical_cast.hpp> | |
#include <boost/algorithm/string.hpp> | |
#include <thrift/transport/TBufferTransports.h> | |
#include <thrift/protocol/TBinaryProtocol.h> | |
#include <thrift/protocol/TCompactProtocol.h> | |
#include "thrift/gen-cpp/test_types.h" | |
#include "thrift/gen-cpp/test_constants.h" | |
#include <capnp/message.h> | |
#include <capnp/serialize.h> | |
#include <cxxopts.hpp> | |
#include "protobuf/test.pb.h" | |
#include "capnproto/test.capnp.h" | |
#include "boost/record.hpp" | |
#include "msgpack/record.hpp" | |
#include "cereal/record.hpp" | |
#include "avro/record.hpp" | |
#include "flatbuffers/test_generated.h" | |
#include "yas/record.hpp" | |
#include "data.hpp" | |
enum class ThriftSerializationProto { Binary, Compact }; | |
struct Args { | |
std::size_t iterations = 0; | |
std::set<std::string> serializers; | |
bool csv = false; | |
}; | |
struct Result { | |
Result(std::string name, std::string version, size_t size, int64_t time) | |
: name(name) | |
, version(version) | |
, size(size) | |
, time(time) | |
{ | |
} | |
Result(std::string name, uint64_t version, size_t size, int64_t time) | |
: name(name) | |
, version(boost::lexical_cast<std::string>(version)) | |
, size(size) | |
, time(time) | |
{ | |
} | |
std::string name; | |
std::string version; | |
size_t size = 0; | |
int64_t time = 0; | |
}; | |
// clang-format off | |
const std::set<std::string> valid_serializers = { | |
"thrift-binary", | |
"thrift-compact", | |
"protobuf", | |
"boost", | |
"msgpack", | |
"cereal", | |
"avro", | |
"capnproto", | |
"flatbuffers", | |
"yas", | |
"yas-compact" | |
}; | |
// clang-format on | |
void | |
print_results(Args args, std::vector<Result> results) | |
{ | |
if (args.csv) { // print CSV header | |
std::cout << "serializer,version,iterations,size,time" << std::endl; | |
} | |
for (const auto& result : results) { | |
if (args.csv) { | |
std::cout << result.name << "," << result.version << "," << args.iterations << "," << result.size << "," | |
<< result.time << std::endl; | |
} else { | |
std::cout << "Serializer: " << result.name << std::endl; | |
std::cout << "Version : " << result.version << std::endl; | |
std::cout << "Iterations: " << args.iterations << std::endl; | |
std::cout << "Size : " << result.size << " bytes" << std::endl; | |
std::cout << "Time : " << result.time << " milliseconds" << std::endl; | |
std::cout << std::endl; | |
} | |
} | |
} | |
void | |
print_supported_serializers() | |
{ | |
std::cerr << std::endl << "Supported serializers are:" << std::endl; | |
for (const auto& s : valid_serializers) { | |
std::cout << " * " << s << std::endl; | |
} | |
std::cout << std::endl; | |
} | |
Args | |
parse_args(int argc, char** argv) | |
{ | |
cxxopts::Options args("benchmark", "Benchmark various C++ serializers"); | |
// clang-format off | |
args.add_options() | |
("h,help", "show this help and exit") | |
("l,list", "show list of supported serializers") | |
("c,csv", "output in CSV format") | |
("i,iterations", "number of serialize/deserialize iterations", cxxopts::value<std::size_t>()) | |
("s,serializers", "comma separated list of serializers to benchmark", cxxopts::value<std::string>()) | |
; | |
// clang-format on | |
Args opts; | |
try { | |
auto parsed_opts = args.parse(argc, argv); | |
if (parsed_opts.count("help")) { | |
std::cout << args.help() << std::endl; | |
exit(EXIT_SUCCESS); | |
} | |
if (parsed_opts.count("list")) { | |
print_supported_serializers(); | |
exit(EXIT_SUCCESS); | |
} | |
if (parsed_opts.count("iterations") == 0) { | |
std::cerr << "Not all required option specified! Please run with -h for available options." << std::endl; | |
exit(EXIT_FAILURE); | |
} | |
if (parsed_opts.count("serializers")) { | |
boost::split(opts.serializers, parsed_opts["serializers"].as<std::string>(), boost::is_any_of(",")); | |
for (const auto& serializer : opts.serializers) { | |
auto exists = valid_serializers.find(serializer); | |
if (exists == valid_serializers.end()) { | |
std::cerr << "Serializer '" << serializer << "' is not supported by this benchmark." << std::endl; | |
print_supported_serializers(); | |
exit(EXIT_FAILURE); | |
} | |
} | |
} | |
opts.iterations = parsed_opts["iterations"].as<std::size_t>(); | |
if (parsed_opts.count("csv")) { | |
opts.csv = true; | |
} | |
} catch (std::exception& exc) { | |
std::cerr << exc.what() << std::endl; | |
exit(EXIT_FAILURE); | |
} | |
return opts; | |
} | |
Result | |
thrift_serialization_test(size_t iterations, ThriftSerializationProto proto = ThriftSerializationProto::Binary) | |
{ | |
using apache::thrift::protocol::TBinaryProtocol; | |
using apache::thrift::protocol::TBinaryProtocolT; | |
using apache::thrift::protocol::TCompactProtocol; | |
using apache::thrift::protocol::TCompactProtocolT; | |
using apache::thrift::transport::TMemoryBuffer; | |
using namespace thrift_test; | |
std::shared_ptr<TMemoryBuffer> buffer1(new TMemoryBuffer()); | |
std::shared_ptr<TMemoryBuffer> buffer2(new TMemoryBuffer()); | |
TBinaryProtocolT<TMemoryBuffer> binary_protocol1(buffer1); | |
TBinaryProtocolT<TMemoryBuffer> binary_protocol2(buffer2); | |
TCompactProtocolT<TMemoryBuffer> compact_protocol1(buffer1); | |
TCompactProtocolT<TMemoryBuffer> compact_protocol2(buffer2); | |
Record r1; | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
r1.ids.push_back(kIntegers[i]); | |
} | |
for (size_t i = 0; i < kStringsCount; i++) { | |
r1.strings.push_back(kStringValue); | |
} | |
std::string serialized; | |
if (proto == ThriftSerializationProto::Binary) { | |
r1.write(&binary_protocol1); | |
} else if (proto == ThriftSerializationProto::Compact) { | |
r1.write(&compact_protocol1); | |
} | |
serialized = buffer1->getBufferAsString(); | |
// check if we can deserialize back | |
Record r2; | |
buffer2->resetBuffer((uint8_t*)serialized.data(), serialized.length()); | |
if (proto == ThriftSerializationProto::Binary) { | |
r2.read(&binary_protocol2); | |
} else if (proto == ThriftSerializationProto::Compact) { | |
r2.read(&compact_protocol2); | |
} | |
if (r1 != r2) { | |
throw std::logic_error("thrift's case: deserialization failed"); | |
} | |
std::string tag; | |
if (proto == ThriftSerializationProto::Binary) { | |
tag = "thrift-binary"; | |
} else if (proto == ThriftSerializationProto::Compact) { | |
tag = "thrift-compact"; | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
buffer1->resetBuffer(); | |
if (proto == ThriftSerializationProto::Binary) { | |
r1.write(&binary_protocol1); | |
} else if (proto == ThriftSerializationProto::Compact) { | |
r1.write(&compact_protocol1); | |
} | |
serialized = buffer1->getBufferAsString(); | |
buffer2->resetBuffer((uint8_t*)serialized.data(), serialized.length()); | |
if (proto == ThriftSerializationProto::Binary) { | |
r2.read(&binary_protocol2); | |
} else if (proto == ThriftSerializationProto::Compact) { | |
r2.read(&compact_protocol2); | |
} | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result(tag, PACKAGE_VERSION, serialized.size(), duration); | |
} | |
Result | |
protobuf_serialization_test(size_t iterations) | |
{ | |
using namespace protobuf_test; | |
Record r1; | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
r1.add_ids(kIntegers[i]); | |
} | |
for (size_t i = 0; i < kStringsCount; i++) { | |
r1.add_strings(kStringValue); | |
} | |
std::string serialized; | |
r1.SerializeToString(&serialized); | |
// check if we can deserialize back | |
Record r2; | |
bool ok = r2.ParseFromString(serialized); | |
if (!ok /*|| r2 != r1*/) { | |
throw std::logic_error("protobuf's case: deserialization failed"); | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
serialized.clear(); | |
r1.SerializeToString(&serialized); | |
r2.ParseFromString(serialized); | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result("protobuf", GOOGLE_PROTOBUF_VERSION, serialized.size(), duration); | |
} | |
Result | |
capnproto_serialization_test(size_t iterations) | |
{ | |
using namespace capnp_test; | |
capnp::MallocMessageBuilder message; | |
Record::Builder r1 = message.getRoot<Record>(); | |
auto ids = r1.initIds(kIntegers.size()); | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
ids.set(i, kIntegers[i]); | |
} | |
auto strings = r1.initStrings(kStringsCount); | |
for (size_t i = 0; i < kStringsCount; i++) { | |
strings.set(i, kStringValue); | |
} | |
kj::ArrayPtr<const kj::ArrayPtr<const capnp::word>> serialized = message.getSegmentsForOutput(); | |
// check if we can deserialize back | |
capnp::SegmentArrayMessageReader reader(serialized); | |
Record::Reader r2 = reader.getRoot<Record>(); | |
if (r2.getIds().size() != kIntegers.size()) { | |
throw std::logic_error("capnproto's case: deserialization failed"); | |
} | |
size_t size = 0; | |
for (auto segment : serialized) { | |
size += segment.asBytes().size(); | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
capnp::MallocMessageBuilder message; | |
Record::Builder r1 = message.getRoot<Record>(); | |
auto ids = r1.initIds(kIntegers.size()); | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
ids.set(i, kIntegers[i]); | |
} | |
auto strings = r1.initStrings(kStringsCount); | |
for (size_t i = 0; i < kStringsCount; i++) { | |
strings.set(i, kStringValue); | |
} | |
serialized = message.getSegmentsForOutput(); | |
capnp::SegmentArrayMessageReader reader(serialized); | |
auto r2 = reader.getRoot<Record>(); | |
(void)r2.getIds().size(); | |
(void)r2.getStrings().size(); | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result("capnproto", CAPNP_VERSION, size, duration); | |
} | |
Result | |
boost_serialization_test(size_t iterations) | |
{ | |
using namespace boost_test; | |
Record r1, r2; | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
r1.ids.push_back(kIntegers[i]); | |
} | |
for (size_t i = 0; i < kStringsCount; i++) { | |
r1.strings.push_back(kStringValue); | |
} | |
std::string serialized; | |
to_string(r1, serialized); | |
from_string(r2, serialized); | |
if (r1 != r2) { | |
throw std::logic_error("boost's case: deserialization failed"); | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
serialized.clear(); | |
to_string(r1, serialized); | |
from_string(r2, serialized); | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result("boost", BOOST_VERSION, serialized.size(), duration); | |
} | |
Result | |
msgpack_serialization_test(size_t iterations) | |
{ | |
using namespace msgpack_test; | |
Record r1, r2; | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
r1.ids.push_back(kIntegers[i]); | |
} | |
for (size_t i = 0; i < kStringsCount; i++) { | |
r1.strings.push_back(kStringValue); | |
} | |
msgpack::sbuffer sbuf; | |
msgpack::pack(sbuf, r1); | |
std::string serialized(sbuf.data(), sbuf.size()); | |
msgpack::object_handle msg = msgpack::unpack(serialized.data(), serialized.size()); | |
msgpack::object obj = msg.get(); | |
obj.convert(r2); | |
if (r1 != r2) { | |
throw std::logic_error("msgpack's case: deserialization failed"); | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
sbuf.clear(); | |
msgpack::pack(sbuf, r1); | |
msgpack::object_handle msg = msgpack::unpack(sbuf.data(), sbuf.size()); | |
msgpack::object obj = msg.get(); | |
obj.convert(r2); | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result("msgpack", msgpack_version(), serialized.size(), duration); | |
} | |
Result | |
cereal_serialization_test(size_t iterations) | |
{ | |
using namespace cereal_test; | |
Record r1, r2; | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
r1.ids.push_back(kIntegers[i]); | |
} | |
for (size_t i = 0; i < kStringsCount; i++) { | |
r1.strings.push_back(kStringValue); | |
} | |
std::string serialized; | |
to_string(r1, serialized); | |
from_string(r2, serialized); | |
if (r1 != r2) { | |
throw std::logic_error("cereal's case: deserialization failed"); | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
serialized.clear(); | |
to_string(r1, serialized); | |
from_string(r2, serialized); | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result("cereal", "", serialized.size(), duration); | |
} | |
Result | |
avro_serialization_test(size_t iterations) | |
{ | |
using namespace avro_test; | |
Record r1, r2; | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
r1.ids.push_back(kIntegers[i]); | |
} | |
for (size_t i = 0; i < kStringsCount; i++) { | |
r1.strings.push_back(kStringValue); | |
} | |
std::unique_ptr<avro::OutputStream> out = avro::memoryOutputStream(); | |
avro::EncoderPtr encoder = avro::binaryEncoder(); | |
encoder->init(*out); | |
avro::encode(*encoder, r1); | |
auto serialized_size = out->byteCount(); | |
std::unique_ptr<avro::InputStream> in = avro::memoryInputStream(*out); | |
avro::DecoderPtr decoder = avro::binaryDecoder(); | |
decoder->init(*in); | |
avro::decode(*decoder, r2); | |
if (r1.ids != r2.ids || r1.strings != r2.strings || r2.ids.size() != kIntegers.size() | |
|| r2.strings.size() != kStringsCount) { | |
throw std::logic_error("avro's case: deserialization failed"); | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
auto out = avro::memoryOutputStream(); | |
auto encoder = avro::binaryEncoder(); | |
encoder->init(*out); | |
avro::encode(*encoder, r1); | |
auto in = avro::memoryInputStream(*out); | |
auto decoder = avro::binaryDecoder(); | |
decoder->init(*in); | |
avro::decode(*decoder, r2); | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result("avro", "", serialized_size, duration); | |
} | |
Result | |
flatbuffers_serialization_test(size_t iterations) | |
{ | |
using namespace flatbuffers_test; | |
std::vector<flatbuffers::Offset<flatbuffers::String>> strings; | |
strings.reserve(kStringsCount); | |
flatbuffers::FlatBufferBuilder builder; | |
for (size_t i = 0; i < kStringsCount; i++) { | |
strings.push_back(builder.CreateString(kStringValue)); | |
} | |
auto ids_vec = builder.CreateVector(kIntegers); | |
auto strings_vec = builder.CreateVector(strings); | |
auto r1 = CreateRecord(builder, ids_vec, strings_vec); | |
builder.Finish(r1); | |
auto p = reinterpret_cast<char*>(builder.GetBufferPointer()); | |
auto sz = builder.GetSize(); | |
std::vector<char> buf(p, p + sz); | |
auto r2 = GetRecord(buf.data()); | |
if (r2->strings()->size() != kStringsCount || r2->ids()->size() != kIntegers.size()) { | |
throw std::logic_error("flatbuffer's case: deserialization failed"); | |
} | |
auto size = builder.GetSize(); | |
builder.ReleaseBufferPointer(); | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
flatbuffers::FlatBufferBuilder builder; | |
strings.clear(); | |
for (size_t i = 0; i < kStringsCount; i++) { | |
strings.push_back(builder.CreateString(kStringValue)); | |
} | |
auto ids_vec = builder.CreateVector(kIntegers); | |
auto strings_vec = builder.CreateVector(strings); | |
auto r1 = CreateRecord(builder, ids_vec, strings_vec); | |
builder.Finish(r1); | |
auto p = reinterpret_cast<char*>(builder.GetBufferPointer()); | |
auto sz = builder.GetSize(); | |
std::vector<char> buf(p, p + sz); | |
auto r2 = GetRecord(buf.data()); | |
(void)r2->ids()[0]; | |
builder.ReleaseBufferPointer(); | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
auto version = FLATBUFFERS_STRING(FLATBUFFERS_VERSION_MAJOR) "." FLATBUFFERS_STRING( | |
FLATBUFFERS_VERSION_MINOR) "." FLATBUFFERS_STRING(FLATBUFFERS_VERSION_REVISION); | |
return Result("flatbuffers", version, size, duration); | |
} | |
template <std::size_t opts> | |
Result | |
yas_serialization_test(size_t iterations) | |
{ | |
using namespace yas_test; | |
Record r1, r2; | |
for (size_t i = 0; i < kIntegers.size(); i++) { | |
r1.ids.push_back(kIntegers[i]); | |
} | |
for (size_t i = 0; i < kStringsCount; i++) { | |
r1.strings.push_back(kStringValue); | |
} | |
std::string serialized; | |
to_string<opts>(r1, serialized); | |
from_string<opts>(r2, serialized); | |
if (r1 != r2) { | |
throw std::logic_error("yas' case: deserialization failed"); | |
} | |
std::string tag; | |
if (opts & yas::compacted) { | |
tag = "yas-compact"; | |
} else { | |
tag = "yas"; | |
} | |
auto start = std::chrono::high_resolution_clock::now(); | |
for (size_t i = 0; i < iterations; i++) { | |
yas::mem_ostream os; | |
yas::binary_oarchive<yas::mem_ostream, opts> oa(os); | |
oa& r1; | |
yas::mem_istream is(os.get_intrusive_buffer()); | |
yas::binary_iarchive<yas::mem_istream, opts> ia(is); | |
ia& r2; | |
} | |
auto finish = std::chrono::high_resolution_clock::now(); | |
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(finish - start).count(); | |
return Result(tag, YAS_VERSION_STRING, serialized.size(), duration); | |
} | |
int | |
main(int argc, char** argv) | |
{ | |
GOOGLE_PROTOBUF_VERIFY_VERSION; | |
auto args = parse_args(argc, argv); | |
/*std::cout << "total size: " << sizeof(kIntegerValue) * kIntegersCount + kStringValue.size() * kStringsCount << | |
* std::endl;*/ | |
std::vector<Result> results; | |
try { | |
if (args.serializers.empty() || args.serializers.find("thrift-binary") != args.serializers.end()) { | |
results.push_back(thrift_serialization_test(args.iterations, ThriftSerializationProto::Binary)); | |
} | |
if (args.serializers.empty() || args.serializers.find("thrift-compact") != args.serializers.end()) { | |
results.push_back(thrift_serialization_test(args.iterations, ThriftSerializationProto::Compact)); | |
} | |
if (args.serializers.empty() || args.serializers.find("protobuf") != args.serializers.end()) { | |
results.push_back(protobuf_serialization_test(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("capnproto") != args.serializers.end()) { | |
results.push_back(capnproto_serialization_test(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("boost") != args.serializers.end()) { | |
results.push_back(boost_serialization_test(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("msgpack") != args.serializers.end()) { | |
results.push_back(msgpack_serialization_test(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("cereal") != args.serializers.end()) { | |
results.push_back(cereal_serialization_test(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("avro") != args.serializers.end()) { | |
results.push_back(avro_serialization_test(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("flatbuffers") != args.serializers.end()) { | |
results.push_back(flatbuffers_serialization_test(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("yas") != args.serializers.end()) { | |
results.push_back(yas_serialization_test<yas::binary | yas::no_header>(args.iterations)); | |
} | |
if (args.serializers.empty() || args.serializers.find("yas-compact") != args.serializers.end()) { | |
results.push_back(yas_serialization_test<yas::binary | yas::no_header | yas::compacted>(args.iterations)); | |
} | |
} catch (std::exception& exc) { | |
std::cerr << "Error: " << exc.what() << std::endl; | |
return EXIT_FAILURE; | |
} | |
print_results(args, results); | |
google::protobuf::ShutdownProtobufLibrary(); | |
return EXIT_SUCCESS; | |
} |