Move "loose object header" ser/de to git-object

The loose header manipulation currently lives in git-pack but it
depends on nothing *of* git-pack.

Move it to git-object (`::encode` and `::decode`), as well as update
the way it's generated: a header should be no more than 28 bytes: 6
bytes for the kind, 20 for the size (log10(2**64) = 19.3), 1 for the
space and 1 for the NUL, so return a `SmallVec<[u8, 28]>` instead of
writing it out directly (this should slightly reduce the amount of IO
when writing out the header to an unbuffered stream), this also avoids
the need for `header_buf` in the traversal state.

In order to generate the header without having to write out the entire
content, add `WriteTo::size()`, the size is (relatively) easy to
compute for all kinds. Ultimately this should avoid having to buffer
or move around the object data before generating the header (though
that's a bit TBD, I don't remember making those changes in git-pack).

This also requires adding size computations to `git_actor::Signature`
and `git_actor::Time`. For the latter the result should be reasonably
efficient[^bithack]. If the time part gets moved to 64b, this should
probably be updated to use a lookup table[^lookup] or even better
`u64::log10` as hopefully it'll be stable by then[^70887].

Also add direct shortcuts to `WriteTo` (to generate a loose header)
and `ObjectRef` (to directly parse a loose object).

Others:

* Lift conversion and header encoding at the start of
  `Sink::write_stream` as they don't seem to depend on the hash kind.
* Alter `loose::Store` to inlin `write_header` but extract the
  creation of the output stream, this reveals that the dispatch on
  hash kind wasn't useful anymore as only the creation of the stream
  needs it (aside from `finalize_object`).

One concern I found when making this change is things are kinda broken
wrt 32/64b: the existing code is a bit mixed between usize and u64,
but tends to store data in buffers which work by usize anyway.

But of course using usize means broken / corrupted files > 4GB on 32b
platforms, which is not great either.

Then again git itself has the same issue except even worse: it uses
`unsigned long` internally, which is not only 32b on 32b platforms
(ILP32) but also on 64 bits windows (LLP64)...

Final note: the `cargo fmt` for commits and tags is *really bad* as it
puts one sub-expression per line, a better alternative might be to
provide size-only and `[inline]` versions of the encoding helpers?

[^bithack]: it's considered a good solution when the input is
    uniformly distributed
    (http://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10Obvious),
    and for us the input is biased toward efficiency: timestamp 1e9 is
    September 2001, and timestamp 1e8 is March 1973, so the vast
    majority of legit commits will take the first branch, and only
    joke commits will fail the second really.
[^lookup]: https://commaok.xyz/post/lookup_tables/ however these solve
    the problem for u32
[^70887]: rust-lang/rust#70887 and
    efficiency improvements could be contributed to the stdlib
    directly.

Author: xmo-odoo

Cargo.lock

@@ -97,18 +97,25 @@ mod write {
         pub fn write_to(&self, out: impl io::Write) -> io::Result<()> {
             self.to_ref().write_to(out)
         }
+        /// Computes the number of bytes necessary to serialize this signature
+        pub fn size(&self) -> usize {
+            self.to_ref().size()
+        }
     }
 
     impl<'a> SignatureRef<'a> {
         /// Serialize this instance to `out` in the git serialization format for actors.
         pub fn write_to(&self, mut out: impl io::Write) -> io::Result<()> {
             out.write_all(validated_token(self.name)?)?;
             out.write_all(SPACE)?;
-            out.write_all(&b"<"[..])?;
+            out.write_all(b"<")?;
             out.write_all(validated_token(self.email)?)?;
-            out.write_all(&b"> "[..])?;
-            self.time.write_to(out)?;
-            Ok(())
+            out.write_all(b"> ")?;
+            self.time.write_to(out)
+        }
+        /// Computes the number of bytes necessary to serialize this signature
+        pub fn size(&self) -> usize {
+            self.name.len() + self.email.len() + self.time.size() + 4
         }
     }
 

git-actor/src/signature/mod.rs

@@ -17,10 +17,10 @@ impl Time {
     pub fn write_to(&self, mut out: impl io::Write) -> io::Result<()> {
         itoa::write(&mut out, self.time)?;
         out.write_all(SPACE)?;
-        out.write_all(&[match self.sign {
-            Sign::Plus => b'+',
-            Sign::Minus => b'-',
-        }])?;
+        out.write_all(match self.sign {
+            Sign::Plus => b"+",
+            Sign::Minus => b"-",
+        })?;
 
         const ZERO: &[u8; 1] = b"0";
 
@@ -40,4 +40,31 @@ impl Time {
         }
         itoa::write(&mut out, minutes).map(|_| ())
     }
+    /// Computes the number of bytes necessary to render this time
+    pub fn size(&self) -> usize {
+        // TODO: this is not year 2038 safe...
+        (if self.time >= 1_000_000_000 {
+            10
+        } else if self.time >= 100_000_000 {
+            9
+        } else if self.time >= 10_000_000 {
+            8
+        } else if self.time >= 1_000_000 {
+            7
+        } else if self.time >= 100_000 {
+            6
+        } else if self.time >= 10_000 {
+            5
+        } else if self.time >= 1_000 {
+            4
+        } else if self.time >= 100 {
+            3
+        } else if self.time >= 10 {
+            2
+        } else {
+            1
+        }) + 2
+            + 2
+            + 2
+    }
 }

git-actor/src/time.rs

@@ -23,11 +23,13 @@ git-hash = { version ="^0.8.0", path = "../git-hash" }
 git-validate = { version ="^0.5.3", path = "../git-validate" }
 git-actor = { version ="^0.6.0", path = "../git-actor" }
 
+btoi = "0.4.2"
+itoa = "0.4.6"
 quick-error = "2.0.0"
 hex = "0.4.2"
 bstr = { version = "0.2.13", default-features = false, features = ["std", "unicode"] }
 nom = { version = "7", default-features = false, features = ["std"]}
-smallvec = "1.4.0"
+smallvec = { version = "1.4.0", features = ["write"] }
 serde = { version = "1.0.114", optional = true, default-features = false, features = ["derive"]}
 
 [dev-dependencies]

git-object/Cargo.toml

@@ -8,6 +8,10 @@ impl<'a> crate::WriteTo for BlobRef<'a> {
         out.write_all(self.data)
     }
 
+    fn size(&self) -> usize {
+        self.data.len()
+    }
+
     fn kind(&self) -> Kind {
         Kind::Blob
     }
@@ -19,6 +23,10 @@ impl crate::WriteTo for Blob {
         self.to_ref().write_to(out)
     }
 
+    fn size(&self) -> usize {
+        self.to_ref().size()
+    }
+
     fn kind(&self) -> Kind {
         Kind::Blob
     }

git-object/src/blob.rs

@@ -1,3 +1,4 @@
+use bstr::ByteSlice;
 use std::io;
 
 use crate::{encode, encode::NL, Commit, CommitRef, Kind};
@@ -21,6 +22,38 @@ impl crate::WriteTo for Commit {
         out.write_all(&self.message)
     }
 
+    fn size(&self) -> usize {
+        let hashsize = self.tree.kind().len_in_hex();
+        b"tree".len()
+            + 1
+            + hashsize
+            + 1
+            + self.parents.iter().count() * (b"parent".len() + 1 + hashsize + 1)
+            + b"author".len()
+            + 1
+            + self.author.size()
+            + 1
+            + b"committer".len()
+            + 1
+            + self.committer.size()
+            + 1
+            + self
+                .encoding
+                .as_ref()
+                .map(|e| b"encoding".len() + 1 + e.len() + 1)
+                .unwrap_or(0)
+            + self
+                .extra_headers
+                .iter()
+                .map(|(name, value)| {
+                    // each header *value* is preceded by a space and followed by a newline
+                    name.len() + value.split_str("\n").map(|s| s.len() + 2).sum::<usize>()
+                })
+                .sum::<usize>()
+            + 1
+            + self.message.len()
+    }
+
     fn kind(&self) -> Kind {
         Kind::Commit
     }
@@ -45,6 +78,38 @@ impl<'a> crate::WriteTo for CommitRef<'a> {
         out.write_all(self.message)
     }
 
+    fn size(&self) -> usize {
+        let hashsize = self.tree().kind().len_in_hex();
+        b"tree".len()
+            + 1
+            + hashsize
+            + 1
+            + self.parents.iter().count() * (b"parent".len() + 1 + hashsize + 1)
+            + b"author".len()
+            + 1
+            + self.author.size()
+            + 1
+            + b"committer".len()
+            + 1
+            + self.committer.size()
+            + 1
+            + self
+                .encoding
+                .as_ref()
+                .map(|e| b"encoding".len() + 1 + e.len() + 1)
+                .unwrap_or(0)
+            + self
+                .extra_headers
+                .iter()
+                .map(|(name, value)| {
+                    // each header *value* is preceded by a space and followed by a newline
+                    name.len() + value.split_str("\n").map(|s| s.len() + 2).sum::<usize>()
+                })
+                .sum::<usize>()
+            + 1
+            + self.message.len()
+    }
+
     fn kind(&self) -> Kind {
         Kind::Commit
     }

git-object/src/commit/write.rs

@@ -1,4 +1,5 @@
-use std::io;
+//! Encoding utilities
+use std::io::{self, Write};
 
 use bstr::{BString, ByteSlice};
 use quick_error::quick_error;
@@ -15,13 +16,28 @@ quick_error! {
     }
 }
 
+macro_rules! check {
+    ($e: expr) => {
+        $e.expect("Writing to a Vec should never fail.")
+    };
+}
+/// Generates a loose header buffer
+pub fn loose_header(kind: crate::Kind, size: usize) -> smallvec::SmallVec<[u8; 28]> {
+    let mut v = smallvec::SmallVec::new();
+    check!(v.write_all(kind.as_bytes()));
+    check!(v.write_all(SPACE));
+    check!(itoa::write(&mut v, size));
+    check!(v.write_all(b"\0"));
+    v
+}
+
 impl From<Error> for io::Error {
     fn from(other: Error) -> io::Error {
         io::Error::new(io::ErrorKind::Other, other)
     }
 }
 
-pub fn header_field_multi_line(name: &[u8], value: &[u8], mut out: impl io::Write) -> io::Result<()> {
+pub(crate) fn header_field_multi_line(name: &[u8], value: &[u8], mut out: impl io::Write) -> io::Result<()> {
     let mut lines = value.as_bstr().split_str(b"\n");
     trusted_header_field(name, lines.next().ok_or(Error::EmptyValue)?, &mut out)?;
     for line in lines {
@@ -32,14 +48,14 @@ pub fn header_field_multi_line(name: &[u8], value: &[u8], mut out: impl io::Writ
     Ok(())
 }
 
-pub fn trusted_header_field(name: &[u8], value: &[u8], mut out: impl io::Write) -> io::Result<()> {
+pub(crate) fn trusted_header_field(name: &[u8], value: &[u8], mut out: impl io::Write) -> io::Result<()> {
     out.write_all(name)?;
     out.write_all(SPACE)?;
     out.write_all(value)?;
     out.write_all(NL)
 }
 
-pub fn trusted_header_signature(
+pub(crate) fn trusted_header_signature(
     name: &[u8],
     value: &git_actor::SignatureRef<'_>,
     mut out: impl io::Write,
@@ -50,14 +66,14 @@ pub fn trusted_header_signature(
     out.write_all(NL)
 }
 
-pub fn trusted_header_id(name: &[u8], value: &git_hash::ObjectId, mut out: impl io::Write) -> io::Result<()> {
+pub(crate) fn trusted_header_id(name: &[u8], value: &git_hash::ObjectId, mut out: impl io::Write) -> io::Result<()> {
     out.write_all(name)?;
-    out.write_all(&SPACE[..])?;
+    out.write_all(SPACE)?;
     value.write_hex_to(&mut out)?;
-    out.write_all(&NL[..])
+    out.write_all(NL)
 }
 
-pub fn header_field(name: &[u8], value: &[u8], out: impl io::Write) -> io::Result<()> {
+pub(crate) fn header_field(name: &[u8], value: &[u8], out: impl io::Write) -> io::Result<()> {
     if value.is_empty() {
         return Err(Error::EmptyValue.into());
     }