feat(quick-protobuf-codec): reduce allocations during encoding

We can reduce the number of allocations during the encoding of messages by not depending on the `Encoder` implementation of `unsigned-varint` but doing the encoding ourselves.

This allows us to directly plug the `BytesMut` into the `Writer` and directly encode into the target buffer. Previously, we were allocating each message as an additional buffer that got immediately thrown-away again.

Related: #4781.

Pull-Request: #4782.

Cargo.toml

@@ -117,7 +117,7 @@ multiaddr = "0.18.1"
 multihash = "0.19.1"
 multistream-select = { version = "0.13.0", path = "misc/multistream-select" }
 prometheus-client = "0.22.0"
-quick-protobuf-codec = { version = "0.3.0", path = "misc/quick-protobuf-codec" }
+quick-protobuf-codec = { version = "0.3.1", path = "misc/quick-protobuf-codec" }
 quickcheck = { package = "quickcheck-ext", path = "misc/quickcheck-ext" }
 rw-stream-sink = { version = "0.4.0", path = "misc/rw-stream-sink" }
 unsigned-varint = { version = "0.8.0" }

misc/quick-protobuf-codec/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.3.1
+
+- Reduce allocations during encoding.
+  See [PR 4782](https://github.com/libp2p/rust-libp2p/pull/4782).
+
 ## 0.3.0
 
 - Update to `asynchronous-codec` `v0.7.0`.

misc/quick-protobuf-codec/Cargo.toml

@@ -3,7 +3,7 @@ name = "quick-protobuf-codec"
 edition = "2021"
 rust-version = { workspace = true }
 description = "Asynchronous de-/encoding of Protobuf structs using asynchronous-codec, unsigned-varint and quick-protobuf."
-version = "0.3.0"
+version = "0.3.1"
 authors = ["Max Inden <mail@max-inden.de>"]
 license = "MIT"
 repository = "https://github.com/libp2p/rust-libp2p"
@@ -14,9 +14,18 @@ categories = ["asynchronous"]
 asynchronous-codec = { workspace = true }
 bytes = { version = "1" }
 thiserror = "1.0"
-unsigned-varint = { workspace = true, features = ["asynchronous_codec"] }
+unsigned-varint = { workspace = true, features = ["std"] }
 quick-protobuf = "0.8"
 
+[dev-dependencies]
+criterion = "0.5.1"
+futures = "0.3.28"
+quickcheck = { workspace = true }
+
+[[bench]]
+name = "codec"
+harness = false
+
 # Passing arguments to the docsrs builder in order to properly document cfg's.
 # More information: https://docs.rs/about/builds#cross-compiling
 [package.metadata.docs.rs]

misc/quick-protobuf-codec/benches/codec.rs

@@ -0,0 +1,28 @@
+use asynchronous_codec::Encoder;
+use bytes::BytesMut;
+use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion};
+use quick_protobuf_codec::{proto, Codec};
+
+pub fn benchmark(c: &mut Criterion) {
+    for size in [1000, 10_000, 100_000, 1_000_000, 10_000_000] {
+        c.bench_with_input(BenchmarkId::new("encode", size), &size, |b, i| {
+            b.iter_batched(
+                || {
+                    let mut out = BytesMut::new();
+                    out.reserve(i + 100);
+                    let codec = Codec::<proto::Message>::new(i + 100);
+                    let msg = proto::Message {
+                        data: vec![0; size],
+                    };
+
+                    (codec, out, msg)
+                },
+                |(mut codec, mut out, msg)| codec.encode(msg, &mut out).unwrap(),
+                BatchSize::SmallInput,
+            );
+        });
+    }
+}
+
+criterion_group!(benches, benchmark);
+criterion_main!(benches);

misc/quick-protobuf-codec/src/generated/mod.rs

@@ -0,0 +1,2 @@
+// Automatically generated mod.rs
+pub mod test;

misc/quick-protobuf-codec/src/generated/test.proto

@@ -0,0 +1,7 @@
+syntax = "proto3";
+
+package test;
+
+message Message {
+    bytes data = 1;
+}

misc/quick-protobuf-codec/src/generated/test.rs

@@ -0,0 +1,47 @@
+// Automatically generated rust module for 'test.proto' file
+
+#![allow(non_snake_case)]
+#![allow(non_upper_case_globals)]
+#![allow(non_camel_case_types)]
+#![allow(unused_imports)]
+#![allow(unknown_lints)]
+#![allow(clippy::all)]
+#![cfg_attr(rustfmt, rustfmt_skip)]
+
+
+use quick_protobuf::{MessageInfo, MessageRead, MessageWrite, BytesReader, Writer, WriterBackend, Result};
+use quick_protobuf::sizeofs::*;
+use super::*;
+
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Debug, Default, PartialEq, Clone)]
+pub struct Message {
+    pub data: Vec<u8>,
+}
+
+impl<'a> MessageRead<'a> for Message {
+    fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result<Self> {
+        let mut msg = Self::default();
+        while !r.is_eof() {
+            match r.next_tag(bytes) {
+                Ok(10) => msg.data = r.read_bytes(bytes)?.to_owned(),
+                Ok(t) => { r.read_unknown(bytes, t)?; }
+                Err(e) => return Err(e),
+            }
+        }
+        Ok(msg)
+    }
+}
+
+impl MessageWrite for Message {
+    fn get_size(&self) -> usize {
+        0
+        + if self.data.is_empty() { 0 } else { 1 + sizeof_len((&self.data).len()) }
+    }
+
+    fn write_message<W: WriterBackend>(&self, w: &mut Writer<W>) -> Result<()> {
+        if !self.data.is_empty() { w.write_with_tag(10, |w| w.write_bytes(&**&self.data))?; }
+        Ok(())
+    }
+}
+