From 367ef3c5915869e6859171a78511b4fa972a2d86 Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 10 May 2024 12:15:01 +0800 Subject: [PATCH] feat: calculate piece digest by blake3 Signed-off-by: Gaius --- Cargo.lock | 46 +++++++++++++++++++------ Cargo.toml | 19 +++++----- dragonfly-client-storage/Cargo.toml | 1 + dragonfly-client-storage/src/content.rs | 11 +++--- dragonfly-client-storage/src/lib.rs | 4 +-- dragonfly-client-util/src/digest/mod.rs | 6 ++++ 6 files changed, 61 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1db14a03..92e0d20b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,12 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247" +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + [[package]] name = "arrayvec" version = "0.7.4" @@ -552,6 +558,19 @@ version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +[[package]] +name = "blake3" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -731,6 +750,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "core-foundation" version = "0.9.4" @@ -919,9 +944,9 @@ dependencies = [ [[package]] name = "dragonfly-api" -version = "2.0.110" +version = "2.0.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "477f701c22a2a3d9f3cfaef271b064ea418b0deb5437115c62217569606b4204" +checksum = "867c43c3b87241820911d2fceb1faf05c6878d01d8ff9ad16524a899db9f77a4" dependencies = [ "prost 0.11.9", "prost-types 0.12.4", @@ -934,7 +959,7 @@ dependencies = [ [[package]] name = "dragonfly-client" -version = "0.1.49" +version = "0.1.50" dependencies = [ "anyhow", "bytes", @@ -997,7 +1022,7 @@ dependencies = [ [[package]] name = "dragonfly-client-backend" -version = "0.1.49" +version = "0.1.50" dependencies = [ "dragonfly-client-core", "futures", @@ -1015,7 +1040,7 @@ dependencies = [ [[package]] name = "dragonfly-client-config" -version = "0.1.49" +version = "0.1.50" dependencies = [ "dragonfly-client-core", "home", @@ -1034,7 +1059,7 @@ dependencies = [ [[package]] name = "dragonfly-client-core" -version = "0.1.49" +version = "0.1.50" dependencies = [ "libloading", "reqwest", @@ -1045,7 +1070,7 @@ dependencies = [ [[package]] name = "dragonfly-client-init" -version = "0.1.49" +version = "0.1.50" dependencies = [ "anyhow", "clap", @@ -1061,9 +1086,10 @@ dependencies = [ [[package]] name = "dragonfly-client-storage" -version = "0.1.49" +version = "0.1.50" dependencies = [ "base16ct", + "blake3", "chrono", "dragonfly-api", "dragonfly-client-config", @@ -1084,7 +1110,7 @@ dependencies = [ [[package]] name = "dragonfly-client-util" -version = "0.1.49" +version = "0.1.50" dependencies = [ "dragonfly-api", "dragonfly-client-core", @@ -1542,7 +1568,7 @@ dependencies = [ [[package]] name = "hdfs" -version = "0.1.49" +version = "0.1.50" dependencies = [ "dragonfly-client-backend", "dragonfly-client-core", diff --git a/Cargo.toml b/Cargo.toml index 80e84d05..164352ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.1.49" +version = "0.1.50" authors = ["The Dragonfly Developers"] homepage = "https://d7y.io/" repository = "https://github.com/dragonflyoss/client.git" @@ -22,15 +22,15 @@ readme = "README.md" edition = "2021" [workspace.dependencies] -dragonfly-client = { path = "dragonfly-client", version = "0.1.49" } -dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.49" } -dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.49" } -dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.49" } -dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.49" } -dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.49" } -dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.49" } +dragonfly-client = { path = "dragonfly-client", version = "0.1.50" } +dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.50" } +dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.50" } +dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.50" } +dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.50" } +dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.50" } +dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.50" } thiserror = "1.0" -dragonfly-api = "2.0.110" +dragonfly-api = "2.0.112" reqwest = { version = "0.11.27", features = ["stream", "native-tls", "default-tls", "rustls-tls"] } rcgen = { version = "0.12.1", features = ["x509-parser"] } hyper = { version = "1.2", features = ["full"] } @@ -50,6 +50,7 @@ rustls = "0.22.2" rustls-pki-types = "1.4.1" rustls-pemfile = "2.1.1" sha2 = "0.10" +blake3 = "1.5.1" uuid = { version = "1.8", features = ["v4"] } hex = "0.4" rocksdb = "0.22.0" diff --git a/dragonfly-client-storage/Cargo.toml b/dragonfly-client-storage/Cargo.toml index ef7894fb..107e6ba6 100644 --- a/dragonfly-client-storage/Cargo.toml +++ b/dragonfly-client-storage/Cargo.toml @@ -24,6 +24,7 @@ prost-wkt-types.workspace = true tokio.workspace = true tokio-util.workspace = true sha2.workspace = true +blake3.workspace = true num_cpus = "1.0" base16ct = { version = "0.2", features = ["alloc"] } diff --git a/dragonfly-client-storage/src/content.rs b/dragonfly-client-storage/src/content.rs index 85c49d8e..8c8d0c39 100644 --- a/dragonfly-client-storage/src/content.rs +++ b/dragonfly-client-storage/src/content.rs @@ -17,7 +17,6 @@ use dragonfly_api::common::v2::Range; use dragonfly_client_config::dfdaemon::Config; use dragonfly_client_core::Result; -use sha2::{Digest, Sha256}; use std::cmp::{max, min}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -245,11 +244,13 @@ impl Content { // Use a buffer to read the piece. let reader = BufReader::with_capacity(self.config.storage.write_buffer_size, reader); - // Sha256 is used to calculate the hash of the piece. - let mut hasher = Sha256::new(); + // Blake3 is used to calculate the hash of the piece. + let mut hasher = blake3::Hasher::new(); // InspectReader is used to calculate the hash of the piece. - let mut tee = InspectReader::new(reader, |bytes| hasher.update(bytes)); + let mut tee = InspectReader::new(reader, |bytes| { + hasher.update(bytes); + }); // Open the file and seek to the offset. let mut f = OpenOptions::new() @@ -279,7 +280,7 @@ impl Content { Ok(WritePieceResponse { length, - hash: base16ct::lower::encode_string(&hash), + hash: base16ct::lower::encode_string(hash.as_bytes()), }) } } diff --git a/dragonfly-client-storage/src/lib.rs b/dragonfly-client-storage/src/lib.rs index f149181a..b7b3c4af 100644 --- a/dragonfly-client-storage/src/lib.rs +++ b/dragonfly-client-storage/src/lib.rs @@ -145,7 +145,7 @@ impl Storage { reader: &mut R, ) -> Result { let response = self.content.write_piece(task_id, offset, reader).await?; - let digest = Digest::new(Algorithm::Sha256, response.hash); + let digest = Digest::new(Algorithm::Blake3, response.hash); self.metadata.download_piece_finished( task_id, @@ -169,7 +169,7 @@ impl Storage { ) -> Result { let response = self.content.write_piece(task_id, offset, reader).await?; let length = response.length; - let digest = Digest::new(Algorithm::Sha256, response.hash); + let digest = Digest::new(Algorithm::Blake3, response.hash); // Check the digest of the piece. if expected_digest != digest.to_string() { diff --git a/dragonfly-client-util/src/digest/mod.rs b/dragonfly-client-util/src/digest/mod.rs index 0772bf27..3edff4bc 100644 --- a/dragonfly-client-util/src/digest/mod.rs +++ b/dragonfly-client-util/src/digest/mod.rs @@ -23,6 +23,9 @@ pub const SEPARATOR: &str = ":"; // Algorithm is a enum of the algorithm that is used to generate digest. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Algorithm { + // Blake3 is blake3 algorithm for generate digest. + Blake3, + // Sha256 is sha256 algorithm for generate digest. Sha256, @@ -35,6 +38,7 @@ impl fmt::Display for Algorithm { // fmt formats the value using the given formatter. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { + Algorithm::Blake3 => write!(f, "blake3"), Algorithm::Sha256 => write!(f, "sha256"), Algorithm::Sha512 => write!(f, "sha512"), } @@ -48,6 +52,7 @@ impl FromStr for Algorithm { // from_str parses a algorithm string. fn from_str(s: &str) -> Result { match s { + "blake3" => Ok(Algorithm::Blake3), "sha256" => Ok(Algorithm::Sha256), "sha512" => Ok(Algorithm::Sha512), _ => Err(format!("invalid digest algorithm: {}", s)), @@ -102,6 +107,7 @@ impl FromStr for Digest { } let algorithm = match parts[0] { + "blake3" => Algorithm::Blake3, "sha256" => Algorithm::Sha256, "sha512" => Algorithm::Sha512, _ => return Err(format!("invalid digest algorithm: {}", parts[0])),