readerwriter: new crate to unify encoding in zkvm/blockchain/p2p

.travis.yml

@@ -10,15 +10,21 @@ matrix:
     ruby: 2.3
     script:
       - DO_NOT_CHECK_HTTP_LINKS=1 ruby check-markdown-links.rb
-  - language: go
-    go: 1.11
-    install: true
-    env: ALLOW_FAILURES=1
+  - language: rust
+    rust: nightly-2020-03-01
+    # per https://levans.fr/rust_travis_cache.html
+    cache:
+      directories:
+        - /home/travis/.cargo
+    before_cache:
+      - rm -rf /home/travis/.cargo/registry
     before_script:
-      - go vet ./slidechain/...
-      - test `gofmt -s -l ./slidechain | grep -vF /vendor/ | wc -l` -eq 0
+    - cd readerwriter
+    - rustup component add rustfmt-preview
     script:
-      - go test -v ./slidechain/... -timeout 60m
+    - cargo fmt --all -- --check
+    - cargo test
+    - RUSTFLAGS="-C opt-level=0" cargo bench "DONOTMATCHANYBENCHMARK"
   - language: rust
     rust: nightly-2020-03-01
     # per https://levans.fr/rust_travis_cache.html

readerwriter/.gitignore

@@ -0,0 +1,2 @@
+target
+Cargo.lock

readerwriter/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "readerwriter"
+version = "0.1.0"
+authors = ["Oleg Andreev <oleganza@gmail.com>"]
+edition = "2018"
+
+[dependencies]
+merlin = {version = "2.0", optional = true }
+bytes = {version = "0.5.4", optional = true }

readerwriter/src/bytes_support.rs

@@ -0,0 +1,101 @@
+//! Reader implementation for Bytes and Buf.
+//! Writer implementation for BytesMut and BufMut.
+
+use crate::{ReadError, Reader, WriteError, Writer};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
+
+impl Reader for Bytes {
+    #[inline]
+    fn read(&mut self, dst: &mut [u8]) -> Result<(), ReadError> {
+        let n = dst.len();
+        if n <= self.remaining_bytes() {
+            self.copy_to_slice(dst);
+            Ok(())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn read_u8(&mut self) -> Result<u8, ReadError> {
+        if self.remaining_bytes() > 0 {
+            Ok(self.get_u8())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn advance(&mut self, n: usize) -> Result<(), ReadError> {
+        if n <= self.remaining_bytes() {
+            <Self as Buf>::advance(self, n);
+            Ok(())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn remaining_bytes(&self) -> usize {
+        self.remaining()
+    }
+}
+
+impl Reader for BytesMut {
+    #[inline]
+    fn read(&mut self, dst: &mut [u8]) -> Result<(), ReadError> {
+        let n = dst.len();
+        if n <= self.remaining_bytes() {
+            self.copy_to_slice(dst);
+            Ok(())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn read_u8(&mut self) -> Result<u8, ReadError> {
+        if self.remaining_bytes() > 0 {
+            Ok(self.get_u8())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn advance(&mut self, n: usize) -> Result<(), ReadError> {
+        if n <= self.remaining_bytes() {
+            <Self as Buf>::advance(self, n);
+            Ok(())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn remaining_bytes(&self) -> usize {
+        self.remaining()
+    }
+}
+
+impl Writer for BytesMut {
+    #[inline]
+    fn write(&mut self, _label: &'static [u8], src: &[u8]) -> Result<(), WriteError> {
+        self.extend_from_slice(src);
+        Ok(())
+    }
+
+    #[inline]
+    fn write_u8(&mut self, _label: &'static [u8], x: u8) -> Result<(), WriteError> {
+        if self.remaining_mut() == 0 {
+            self.reserve(1);
+        }
+        self.put_u8(x);
+        Ok(())
+    }
+
+    #[inline]
+    fn remaining_capacity(&self) -> usize {
+        usize::max_value()
+    }
+}

readerwriter/src/lib.rs

@@ -0,0 +1,15 @@
+mod reader;
+mod writer;
+
+pub use reader::{ReadError, Reader};
+pub use writer::{WriteError, Writer};
+
+#[cfg(feature = "merlin")]
+mod merlin_support;
+#[cfg(feature = "merlin")]
+pub use merlin_support::*;
+
+#[cfg(feature = "bytes")]
+mod bytes_support;
+#[cfg(feature = "bytes")]
+pub use bytes_support::*;

readerwriter/src/merlin_support.rs

@@ -0,0 +1,17 @@
+//! Writer implementation for merlin::Transcript.
+
+use crate::{WriteError, Writer};
+use merlin::Transcript;
+
+impl Writer for Transcript {
+    #[inline]
+    fn write(&mut self, label: &'static [u8], src: &[u8]) -> Result<(), WriteError> {
+        self.append_message(label, src);
+        Ok(())
+    }
+
+    #[inline]
+    fn remaining_capacity(&self) -> usize {
+        usize::max_value()
+    }
+}

readerwriter/src/reader.rs

@@ -0,0 +1,171 @@
+/// Error kinds returns by the reader.
+#[derive(Debug, Clone, PartialEq)]
+pub enum ReadError {
+    InsufficientBytes,
+    TrailingBytes,
+    InvalidFormat,
+}
+
+/// An interface for reading binary data.
+pub trait Reader {
+    /// Copies bytes into a slice. If there is not enough bytes available,
+    /// does not consume any byte and returns ReadError::InsufficientBytes.
+    fn read(&mut self, dst: &mut [u8]) -> Result<(), ReadError>;
+
+    /// Advances the internal cursor by the number of bytes.
+    /// If there is not enough bytes, does nothing and returns ReadError::InsufficientBytes.
+    fn advance(&mut self, cnt: usize) -> Result<(), ReadError>;
+
+    /// Returns remaining number of bytes available for reading.
+    fn remaining_bytes(&self) -> usize;
+
+    /// Wraps the reading logic in a block that checks that all bytes have been read.
+    /// If some are left unread, returns `Err(From<ReadError::TrailingBytes>)`.
+    /// Use method `skip_trailing_bytes` to ignore trailing bytes.
+    #[inline]
+    fn read_all<F, T, E>(&mut self, closure: F) -> Result<T, E>
+    where
+        F: FnOnce(&mut Self) -> Result<T, E>,
+        E: From<ReadError>,
+    {
+        let result = closure(self)?;
+        if self.remaining_bytes() != 0 {
+            return Err(ReadError::TrailingBytes.into());
+        }
+        Ok(result)
+    }
+
+    /// Marks remaining unread bytes as read so that `read_all` does not fail.
+    /// After calling this method, no more bytes can be read.
+    #[inline]
+    fn skip_trailing_bytes(&mut self) -> usize {
+        let rem = self.remaining_bytes();
+        self.advance(rem)
+            .expect("Reader::advance(remaining()) should never fail");
+        rem
+    }
+
+    /// Reads a single byte.
+    #[inline]
+    fn read_u8(&mut self) -> Result<u8, ReadError> {
+        let mut buf = [0u8; 1];
+        self.read(&mut buf)?;
+        Ok(buf[0])
+    }
+
+    /// Reads a 4-byte LE32 integer.
+    #[inline]
+    fn read_u32(&mut self) -> Result<u32, ReadError> {
+        let mut buf = [0u8; 4];
+        self.read(&mut buf)?;
+        Ok(u32::from_le_bytes(buf))
+    }
+
+    /// Reads an 8-byte LE64 integer.
+    #[inline]
+    fn read_u64(&mut self) -> Result<u64, ReadError> {
+        let mut buf = [0u8; 8];
+        self.read(&mut buf)?;
+        Ok(u64::from_le_bytes(buf))
+    }
+
+    /// Reads a 32-byte string.
+    #[inline]
+    fn read_u8x32(&mut self) -> Result<[u8; 32], ReadError> {
+        let mut buf = [0u8; 32];
+        self.read(&mut buf)?;
+        Ok(buf)
+    }
+
+    /// Reads a 64-byte string.
+    #[inline]
+    fn read_u8x64(&mut self) -> Result<[u8; 64], ReadError> {
+        let mut buf = [0u8; 64];
+        self.read(&mut buf)?;
+        Ok(buf)
+    }
+
+    /// Reads a vector of bytes with the required length.
+    #[inline]
+    fn read_vec(&mut self, len: usize) -> Result<Vec<u8>, ReadError> {
+        if self.remaining_bytes() < len {
+            // this early check is to avoid allocating a vector
+            // if we don't have enough data.
+            return Err(ReadError::InsufficientBytes);
+        }
+        let mut vec = Vec::with_capacity(len);
+        vec.resize(len, 0u8);
+        self.read(&mut vec)?;
+        Ok(vec)
+    }
+
+    /// Reads a vector of items with the required count and a minimum item size.
+    #[inline]
+    fn read_vec_with<T, E>(
+        &mut self,
+        len: usize,
+        min_item_size: usize,
+        closure: impl Fn(&mut Self) -> Result<T, E>,
+    ) -> Result<Vec<T>, E>
+    where
+        E: From<ReadError>,
+    {
+        if len > self.remaining_bytes() / min_item_size {
+            // this early check is to avoid allocating a vector
+            // if we don't have enough data.
+            return Err(ReadError::InsufficientBytes.into());
+        }
+        let mut vec = Vec::with_capacity(len);
+        for _ in 0..len {
+            vec.push(closure(self)?);
+        }
+        Ok(vec)
+    }
+}
+
+impl Reader for &[u8] {
+    #[inline]
+    fn read(&mut self, dst: &mut [u8]) -> Result<(), ReadError> {
+        let n = dst.len();
+        if n <= self.len() {
+            let (a, b) = self.split_at(n);
+            if n == 1 {
+                dst[0] = a[0];
+            } else {
+                dst.copy_from_slice(a);
+            }
+            *self = b;
+            Ok(())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn read_u8(&mut self) -> Result<u8, ReadError> {
+        if self.len() > 0 {
+            let x = self[0];
+            let (_, rest) = self.split_at(1);
+            *self = rest;
+            Ok(x)
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn advance(&mut self, n: usize) -> Result<(), ReadError> {
+        if n <= self.len() {
+            let (_, b) = self.split_at(n);
+            *self = b;
+            Ok(())
+        } else {
+            Err(ReadError::InsufficientBytes)
+        }
+    }
+
+    #[inline]
+    fn remaining_bytes(&self) -> usize {
+        self.len()
+    }
+}