From bd2339cfbb704fb85098766a20b01ec13fe0fbc9 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Wed, 27 Nov 2024 21:18:35 +0800 Subject: [PATCH 1/5] fix(`to_workdir_path`): support "." & "./", replace crate `path_abs` with `path_absolutize` Signed-off-by: Qihang Cai --- libra/Cargo.toml | 2 +- libra/src/command/status.rs | 1 - libra/src/utils/lfs.rs | 1 - libra/src/utils/util.rs | 35 ++++++++++++++++++++--------------- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/libra/Cargo.toml b/libra/Cargo.toml index 31fb67e12..cdd1c5cd6 100644 --- a/libra/Cargo.toml +++ b/libra/Cargo.toml @@ -27,7 +27,7 @@ lazy_static = { workspace = true } lru-mem = "0.3.0" mercury = { workspace = true } once_cell = "1.19.0" -path_abs = "0.5.1" +path-absolutize = "3.1.1" pathdiff = "0.2.1" regex = { workspace = true } reqwest = { workspace = true, features = ["stream", "json"] } diff --git a/libra/src/command/status.rs b/libra/src/command/status.rs index 3303e7151..284e96e62 100644 --- a/libra/src/command/status.rs +++ b/libra/src/command/status.rs @@ -2,7 +2,6 @@ use std::collections::HashSet; use std::path::PathBuf; use colored::Colorize; -use path_abs::PathInfo; use mercury::internal::object::commit::Commit; use mercury::internal::object::tree::Tree; diff --git a/libra/src/utils/lfs.rs b/libra/src/utils/lfs.rs index ba5e03ce2..78bd8971f 100644 --- a/libra/src/utils/lfs.rs +++ b/libra/src/utils/lfs.rs @@ -2,7 +2,6 @@ use crate::utils::path_ext::PathExt; use crate::utils::{path, util}; use lazy_static::lazy_static; use mercury::internal::index::Index; -use path_abs::{PathInfo, PathOps}; use regex::Regex; use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, CONTENT_TYPE}; use ring::digest::{Context, SHA256}; diff --git a/libra/src/utils/util.rs b/libra/src/utils/util.rs index 979cad064..f38fbaef9 100644 --- a/libra/src/utils/util.rs +++ b/libra/src/utils/util.rs @@ -1,7 +1,7 @@ -use path_abs::{PathAbs, PathInfo}; use std::collections::HashSet; use std::io::Write; use std::path::{Path, PathBuf}; +use path_absolutize::*; use std::{env, fs, io}; use indicatif::{ProgressBar, ProgressStyle}; use mercury::hash::SHA1; @@ -99,8 +99,8 @@ where P: AsRef, B: AsRef, { - let path_abs = PathAbs::new(path.as_ref()).unwrap(); // prefix: '\\?\' on Windows - let parent_abs = PathAbs::new(parent.as_ref()).unwrap(); + let path_abs = path.as_ref().absolutize().unwrap(); + let parent_abs = parent.as_ref().absolutize().unwrap(); path_abs.starts_with(parent_abs) } @@ -145,15 +145,12 @@ where P: AsRef, B: AsRef, { - let path_abs = PathAbs::new(path.as_ref()).unwrap(); // prefix: '\\?\' on Windows - let base_abs = PathAbs::new(base.as_ref()).unwrap(); - if cfg!(windows) { - assert_eq!( - // just little check - path_abs.to_str().unwrap().starts_with(r"\\?\"), - base_abs.to_str().unwrap().starts_with(r"\\?\") - ) - } + // × crate `PathAbs` is NOT good enough + // 1. `PathAbs::new` can not handle `.` or `./`, all return "" + // 2. `PathAbs::new` generate prefix: '\\?\' on Windows + // So, we replace it with `path_absolutize` √ + let path_abs = path.as_ref().absolutize().unwrap(); + let base_abs = base.as_ref().absolutize().unwrap(); if let Some(rel_path) = pathdiff::diff_paths(path_abs, base_abs) { rel_path } else { @@ -276,7 +273,7 @@ pub fn is_empty_dir(dir: &Path) -> bool { } pub fn is_cur_dir(dir: &Path) -> bool { - PathAbs::new(dir).unwrap() == PathAbs::new(cur_dir()).unwrap() + dir.absolutize().unwrap() == cur_dir().absolutize().unwrap() } /// transform path to string, use '/' as separator even on windows @@ -358,12 +355,20 @@ mod test { assert!(is_sub_path("src/main.rs", "src")); assert!(is_sub_path("src/main.rs", "src/")); assert!(is_sub_path("src/main.rs", "src/main.rs")); + assert!(is_sub_path("src/main.rs", ".")); + } + + #[test] + fn test_to_relative() { + assert_eq!(to_relative("src/main.rs", "src"), PathBuf::from("main.rs")); + assert_eq!(to_relative(".", "src"), PathBuf::from("..")); } #[tokio::test] async fn test_to_workdir_path() { test::setup_with_new_libra().await; - let workdir_path = to_workdir_path("src/main.rs"); - assert_eq!(workdir_path, PathBuf::from("src/main.rs")); + assert_eq!(to_workdir_path("./src/abc/../main.rs"), PathBuf::from("src/main.rs")); + assert_eq!(to_workdir_path("."), PathBuf::from("")); + assert_eq!(to_workdir_path("./"), PathBuf::from("")); } } From 88f7508da31ae2ee27706c4b1316e50ca3fc6545 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Thu, 28 Nov 2024 17:34:00 +0800 Subject: [PATCH 2/5] fix(`to_relative`): return "." if path == base, & `to_workdir_path(".") == "."` Signed-off-by: Qihang Cai --- libra/src/command/diff.rs | 14 ++++---------- libra/src/utils/path_ext.rs | 3 +++ libra/src/utils/util.rs | 14 +++++++++++--- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/libra/src/command/diff.rs b/libra/src/command/diff.rs index 1cfa6accd..20476b196 100644 --- a/libra/src/command/diff.rs +++ b/libra/src/command/diff.rs @@ -28,6 +28,8 @@ use crate::{ #[cfg(unix)] use std::process::{Command, Stdio}; +use crate::utils::path_ext::PathExt; + #[derive(Parser, Debug)] pub struct DiffArgs { #[clap(long, help = "Old commit, defaults is staged or HEAD")] @@ -123,15 +125,7 @@ pub async fn execute(args: DiffArgs) { .pathspec .iter() .map(|s| { - let path = { - let _path = PathBuf::from(s); - if _path.is_absolute() { - _path - } else { - std::env::current_dir().unwrap().join(_path) // proces path such as `./` - } - }; - util::to_workdir_path(&path) + util::to_workdir_path(s) }) .collect(); @@ -182,7 +176,7 @@ pub async fn diff( // filter files, cross old and new files, and pathspec for (new_file, new_hash) in new_blobs { // if new_file did't start with any path in filter, skip it - if !filter.is_empty() && !filter.iter().any(|path| new_file.starts_with(path)) { + if !filter.is_empty() && !filter.iter().any(|path| new_file.sub_of(path)) { continue; } match old_blobs.get(&new_file) { diff --git a/libra/src/utils/path_ext.rs b/libra/src/utils/path_ext.rs index d5c6cf1d1..a987b2710 100644 --- a/libra/src/utils/path_ext.rs +++ b/libra/src/utils/path_ext.rs @@ -34,6 +34,9 @@ impl PathExt for PathBuf { util::workdir_to_current(self) } + /// Check if `self` is a sub path (child) of `parent`
+ /// Simply convert to absolute path (to current dir) and call `starts_with` + /// - aka: "src/main.rs" is a sub path of "src/" fn sub_of(&self, parent: &Path) -> bool { util::is_sub_path(self, parent) } diff --git a/libra/src/utils/util.rs b/libra/src/utils/util.rs index f38fbaef9..91443181a 100644 --- a/libra/src/utils/util.rs +++ b/libra/src/utils/util.rs @@ -99,6 +99,8 @@ where P: AsRef, B: AsRef, { + // to absolute, just for clear redundant `..` `.` in the path + // may generate wrong intermediate path, but the final result is correct (after `starts_with`) let path_abs = path.as_ref().absolutize().unwrap(); let parent_abs = parent.as_ref().absolutize().unwrap(); path_abs.starts_with(parent_abs) @@ -140,6 +142,7 @@ where } /// `path` & `base` must be absolute or relative (to current dir) +///
return "." if `path` == `base` pub fn to_relative(path: P, base: B) -> PathBuf where P: AsRef, @@ -152,7 +155,11 @@ where let path_abs = path.as_ref().absolutize().unwrap(); let base_abs = base.as_ref().absolutize().unwrap(); if let Some(rel_path) = pathdiff::diff_paths(path_abs, base_abs) { - rel_path + if rel_path.to_string_lossy() == "" { + PathBuf::from(".") + } else { + rel_path + } } else { panic!( "fatal: path {:?} cannot convert to relative based on {:?}", @@ -368,7 +375,8 @@ mod test { async fn test_to_workdir_path() { test::setup_with_new_libra().await; assert_eq!(to_workdir_path("./src/abc/../main.rs"), PathBuf::from("src/main.rs")); - assert_eq!(to_workdir_path("."), PathBuf::from("")); - assert_eq!(to_workdir_path("./"), PathBuf::from("")); + assert_eq!(to_workdir_path("."), PathBuf::from(".")); + assert_eq!(to_workdir_path("./"), PathBuf::from(".")); + assert_eq!(to_workdir_path(""), PathBuf::from(".")); } } From a6c5ee28becbafa4899bfdce04600221cc2d44b5 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 29 Nov 2024 16:59:08 +0800 Subject: [PATCH 3/5] improve(config): support override nested key by env vars (with separator `__`) Signed-off-by: Qihang Cai --- common/src/config.rs | 2 +- docs/development.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/config.rs b/common/src/config.rs index 1d16836d0..329d144fb 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -27,7 +27,7 @@ impl Config { pub fn new(path: &str) -> Result { let builder = c::Config::builder() .add_source(c::File::new(path, FileFormat::Toml)) - .add_source(c::Environment::with_prefix("mega")); // e.g. MEGA_BASE_DIR == base_dir + .add_source(c::Environment::with_prefix("mega").prefix_separator("_").separator("__")); // e.g. MEGA_BASE_DIR == base_dir // support ${} variable substitution let config = variable_placeholder_substitute(builder); diff --git a/docs/development.md b/docs/development.md index 4243fc2df..33aaa1f4d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -226,7 +226,7 @@ Config `confg.toml` file for the Mega project. ### Enhance - You can use environment variables starting with `MEGA_` to override the configuration in `config.toml`. - like `MEGA_BASE_DIR` to override `base_dir`. // with `env::set_var()` - - but it seems only not available for nested keys, like `log.log_path`. + - use separator `__` (2 \* `_`) for nested keys, like `MEGA_LOG__LOG_PATH` for `log.log_path`. - Support `${}` syntax to reference other keys in the same file. - like `log_path = "${base_dir}/logs"`, `${base_dir}` will be replaced by the value of `base_dir` - or `key = "${xxx.yyy}/zzz"` (prefix `xxx.` can't be omitted) From b6b93098785b8921c2fb66b951f83c9376acbd03 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 29 Nov 2024 17:07:49 +0800 Subject: [PATCH 4/5] feat(mega-tests): enable http_auth in libra-mega tests Signed-off-by: Qihang Cai --- libra/src/command/mod.rs | 13 ++++- libra/src/internal/protocol/https_client.rs | 8 ++- mega/Cargo.toml | 1 + mega/tests/lfs_test.rs | 57 ++++++++++----------- 4 files changed, 47 insertions(+), 32 deletions(-) diff --git a/libra/src/command/mod.rs b/libra/src/command/mod.rs index d1ca948a3..d1906a630 100644 --- a/libra/src/command/mod.rs +++ b/libra/src/command/mod.rs @@ -63,10 +63,21 @@ fn ask_username_password() -> (String, String) { let mut username = String::new(); io::stdin().read_line(&mut username).unwrap(); username = username.trim().to_string(); + tracing::debug!("username: {}", username); print!("password: "); io::stdout().flush().unwrap(); - let password = read_password().unwrap(); // hide password + let password = if std::env::var("LIBRA_NO_HIDE_PASSWORD").is_ok() { + // for test + let mut password = String::new(); + io::stdin().read_line(&mut password).unwrap(); + password = password.trim().to_string(); + tracing::debug!("password: {}", password); + password + } else { + // error in test environment: "No such device or address" + read_password().unwrap() // hide password + }; (username, password) } diff --git a/libra/src/internal/protocol/https_client.rs b/libra/src/internal/protocol/https_client.rs index 2955d205b..ba7c2ca81 100644 --- a/libra/src/internal/protocol/https_client.rs +++ b/libra/src/internal/protocol/https_client.rs @@ -43,14 +43,18 @@ pub struct BasicAuth { pub(crate) username: String, pub(crate) password: String, } - +static AUTH: Mutex> = Mutex::new(None); impl BasicAuth { + /// set username & password manually + pub async fn set_auth(auth: BasicAuth) { + AUTH.lock().unwrap().replace(auth); + } + /// send request with basic auth, retry 3 times pub async fn send(request_builder: impl Fn() -> Fut) -> Result where Fut: std::future::Future, { - static AUTH: Mutex> = Mutex::new(None); const MAX_TRY: usize = 3; let mut res; let mut try_cnt = 0; diff --git a/mega/Cargo.toml b/mega/Cargo.toml index 86733545f..925fd669c 100644 --- a/mega/Cargo.toml +++ b/mega/Cargo.toml @@ -39,6 +39,7 @@ ctrlc = { workspace = true } tempfile = { workspace = true } serial_test = "3.1.1" lazy_static = {workspace = true} +assert_cmd = "2.0.16" [build-dependencies] shadow-rs = { workspace = true } diff --git a/mega/tests/lfs_test.rs b/mega/tests/lfs_test.rs index 6cac27f86..240592f93 100644 --- a/mega/tests/lfs_test.rs +++ b/mega/tests/lfs_test.rs @@ -46,41 +46,36 @@ fn check_git_lfs() -> bool { status.success() } -fn run_cmd(program: &str, args: &[&str]) { - let output = Command::new(program) - .args(args) - .output() - .unwrap(); - - let status = output.status; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - if !status.success() { - panic!( - "Command failed: {} {}\nStatus: {}\nStdout: {}\nStderr: {}", - program, - args.join(" "), - status, - stdout, - stderr - ); - } else { - println!( - "Command success: {} {}\nStatus: {}\nStdout: {}", - program, - args.join(" "), - status, - stdout, - ); +fn run_cmd(program: &str, args: &[&str], stdin: Option<&str>, envs: Option>) { + let mut cmd = assert_cmd::Command::new(program); + let mut cmd = cmd.args(args); + if let Some(stdin) = stdin { + cmd = cmd.write_stdin(stdin); } + if let Some(envs) = envs { + cmd = cmd.envs(envs); + } + let assert = cmd.assert().success(); + let output = assert.get_output(); + + println!("Command success: {} {}\nStatus: {}\nStdout: {}", + program, + args.join(" "), + output.status, + String::from_utf8_lossy(&output.stdout), + ); } fn run_git_cmd(args: &[&str]) { - run_cmd("git", args); + run_cmd("git", args, None, None); } fn run_libra_cmd(args: &[&str]) { - run_cmd(LIBRA.to_str().unwrap(), args); + run_cmd(LIBRA.to_str().unwrap(), args, None, None); +} + +fn run_libra_cmd_with_stdin(args: &[&str], stdin: Option<&str>, envs: Option>) { + run_cmd(LIBRA.to_str().unwrap(), args, stdin, envs); } fn is_port_in_use(port: u16) -> bool { @@ -185,7 +180,9 @@ fn libra_lfs_push(url: &str) -> io::Result<()> { // try lock API run_libra_cmd(&["lfs", "lock", "large_file.bin"]); // push to mega server - run_libra_cmd(&["push", "mega", "master"]); + run_libra_cmd_with_stdin(&["push", "mega", "master"], + Some("mega\nmega"), // basic auth + Some(vec![("LIBRA_NO_HIDE_PASSWORD", "1")])); Ok(()) } @@ -242,6 +239,7 @@ fn lfs_split_with_libra() { } let mega_dir = TempDir::new().unwrap(); + env::set_var("MEGA_authentication__enable_http_auth", "true"); // start mega server at background (new process) let mut mega = run_mega_server(mega_dir.path()); @@ -249,6 +247,7 @@ fn lfs_split_with_libra() { libra_lfs_push(url).expect("(libra)Failed to push large file to mega server"); libra_lfs_clone(url).expect("(libra)Failed to clone large file from mega server"); + env::set_var("MEGA_authentication__enable_http_auth", "false"); // avoid affecting other tests mega.kill().expect("Failed to kill mega server"); thread::sleep(Duration::from_secs(1)); // wait for server to stop, avoiding affecting other tests } \ No newline at end of file From 5e29be358ac47e26a167e5733b7f1552c1548128 Mon Sep 17 00:00:00 2001 From: Qihang Cai Date: Fri, 29 Nov 2024 17:22:54 +0800 Subject: [PATCH 5/5] improve(mega-tests): set username & password manually via env vars, ensure it works Signed-off-by: Qihang Cai --- mega/tests/lfs_test.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mega/tests/lfs_test.rs b/mega/tests/lfs_test.rs index 240592f93..5c37f6eab 100644 --- a/mega/tests/lfs_test.rs +++ b/mega/tests/lfs_test.rs @@ -181,7 +181,7 @@ fn libra_lfs_push(url: &str) -> io::Result<()> { run_libra_cmd(&["lfs", "lock", "large_file.bin"]); // push to mega server run_libra_cmd_with_stdin(&["push", "mega", "master"], - Some("mega\nmega"), // basic auth + Some("mega\nmega"), // basic auth, can be overridden by env var Some(vec![("LIBRA_NO_HIDE_PASSWORD", "1")])); Ok(()) @@ -220,6 +220,7 @@ fn lfs_split_with_git() { assert!(check_git_lfs(), "git lfs is not installed"); let mega_dir = TempDir::new().unwrap(); + env::set_var("MEGA_authentication__enable_http_auth", "false"); // no need for git // start mega server at background (new process) let mut mega = run_mega_server(mega_dir.path()); @@ -240,6 +241,9 @@ fn lfs_split_with_libra() { let mega_dir = TempDir::new().unwrap(); env::set_var("MEGA_authentication__enable_http_auth", "true"); + env::set_var("MEGA_authentication__enable_test_user", "true"); + env::set_var("MEGA_authentication__test_user_name", "mega"); + env::set_var("MEGA_authentication__test_user_token", "mega"); // start mega server at background (new process) let mut mega = run_mega_server(mega_dir.path());