diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 70071d08d..59188463c 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -3,57 +3,56 @@ name = "libra" version = "0.1.0" edition = "2021" -[[bin]] -name = "libra" -path = "src/main.rs" +[features] +default = [] +p2p = ["gemini"] [dependencies] -mercury = { workspace = true } +anyhow = { workspace = true } +byte-unit = "5.1.4" +byteorder = "1.5.0" +bytes = { workspace = true } ceres = { workspace = true } -gemini = { workspace = true } - -sea-orm = { workspace = true, features = [ - "sqlx-sqlite", - "runtime-tokio-rustls", - "macros", -] } -tokio = { workspace = true, features = ["rt-multi-thread", "rt", "macros"] } +chrono = { workspace = true } clap = { workspace = true, features = ["derive"] } +color-backtrace = "0.6.1" +colored = { workspace = true } flate2 = { workspace = true } # add features = ["zlib"] if slow -tracing = { workspace = true } -tracing-subscriber ={ workspace = true } -sha1 = { workspace = true } -bytes = { workspace = true } -chrono = { workspace = true } futures = { workspace = true } -reqwest = { workspace = true, features = ["stream", "json"] } -tokio-util = { version = "0.7.11", features = ["io"] } -color-backtrace = "0.6.1" -colored = "2.1.0" -byteorder = "1.5.0" -path_abs = "0.5.1" -pathdiff = "0.2.1" -url = "2.5.0" -futures-util = "0.3.30" -rpassword = "7.3.1" +futures-util = { workspace = true } +gemini = { workspace = true, optional = true } +hex = { workspace = true } indicatif = "0.17.8" -wax = "0.6.0" lazy_static = { workspace = true } +lru-mem = "0.3.0" +mercury = { workspace = true } +once_cell = "1.19.0" +path_abs = "0.5.1" +pathdiff = "0.2.1" regex = { workspace = true } -ring = "0.17.8" -hex = { workspace = true } +reqwest = { workspace = true, features = ["stream", "json"] } +ring = { workspace = true } +rpassword = "7.3.1" +scopeguard = "1.2.0" +sea-orm = { workspace = true, features = [ + "sqlx-sqlite", + "runtime-tokio-rustls", + "macros", +] } serde = { workspace = true } serde_json = { workspace = true } -once_cell = "1.19.0" -byte-unit = "5.1.4" -scopeguard = "1.2.0" -lru-mem = "0.3.0" -anyhow = { workspace = true } +sha1 = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "rt", "macros"] } +tokio-util = { version = "0.7.11", features = ["io"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +url = "2.5.3" +wax = "0.6.0" [target.'cfg(unix)'.dependencies] # only on Unix pager = "0.16.0" [dev-dependencies] +tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "process"] } tracing-test = "0.2.4" -tempfile = { workspace = true } diff --git a/libra/README.md b/libra/README.md index dff111a53..9cf810a77 100644 --- a/libra/README.md +++ b/libra/README.md @@ -81,7 +81,7 @@ achieving unified management. ### Others - [ ] `.gitignore` - [x] `.gitattributes` (only for `lfs` now) -- [x] `LFS` (embedded) +- [x] `LFS` (embedded, with p2p feature) - [ ] `ssh` ## Development diff --git a/libra/src/internal/protocol/lfs_client.rs b/libra/src/internal/protocol/lfs_client.rs index 85c914e35..5b2af350a 100644 --- a/libra/src/internal/protocol/lfs_client.rs +++ b/libra/src/internal/protocol/lfs_client.rs @@ -428,14 +428,41 @@ impl LFSClient { } } - #[allow(clippy::type_complexity)] + /// Only for MonoRepo (mega) + async fn fetch_chunks(&self, obj_link: &str) -> Result, ()> { + let mut url = Url::parse(obj_link).unwrap(); + let path = url.path().trim_end_matches('/'); + url.set_path(&(path.to_owned() + "/chunks")); // reserve query params (for GitHub link) + + let request = self.client.get(url); + let resp = request.send().await.unwrap(); + let code = resp.status(); + if code == StatusCode::NOT_FOUND || code == StatusCode::FORBIDDEN { // GitHub maybe return 403 + tracing::info!("Remote LFS Server not support Chunks API, or forbidden."); + return Err(()); + } else if !code.is_success() { + tracing::debug!("fatal: LFS get chunk hrefs failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); + return Err(()); + } + let mut res = resp.json::().await.unwrap(); + // sort by offset + res.chunks.sort_by(|a, b| a.offset.cmp(&b.offset)); + Ok(res.chunks) + } +} + +#[cfg(feature="p2p")] +type Reporter = (dyn FnMut(f64) -> anyhow::Result<()> + Send); +#[cfg(feature="p2p")] +impl LFSClient{ + /// download (GET) one LFS file peer-to-peer pub async fn download_object_p2p( &self, file_uri: &str, // p2p protocol path: impl AsRef, mut reporter: Option<( - &mut (dyn FnMut(f64) -> anyhow::Result<()> + Send), // progress callback + &mut Reporter, // progress callback f64 // step )>) -> anyhow::Result<()> { @@ -549,28 +576,6 @@ impl LFSClient { } Ok(buffer) } - - /// Only for MonoRepo (mega) - async fn fetch_chunks(&self, obj_link: &str) -> Result, ()> { - let mut url = Url::parse(obj_link).unwrap(); - let path = url.path().trim_end_matches('/'); - url.set_path(&(path.to_owned() + "/chunks")); // reserve query params (for GitHub link) - - let request = self.client.get(url); - let resp = request.send().await.unwrap(); - let code = resp.status(); - if code == StatusCode::NOT_FOUND || code == StatusCode::FORBIDDEN { // GitHub maybe return 403 - tracing::info!("Remote LFS Server not support Chunks API, or forbidden."); - return Err(()); - } else if !code.is_success() { - tracing::debug!("fatal: LFS get chunk hrefs failed. Status: {}, Message: {}", code, resp.text().await.unwrap()); - return Err(()); - } - let mut res = resp.json::().await.unwrap(); - // sort by offset - res.chunks.sort_by(|a, b| a.offset.cmp(&b.offset)); - Ok(res.chunks) - } } // LFS locks API @@ -715,6 +720,7 @@ mod tests { } #[tokio::test] + #[cfg(feature="p2p")] #[ignore] // need to start local mega server async fn test_download_chunk() { let client = LFSClient::from_url(&Url::parse("http://localhost:8000").unwrap());