From 5b680a573cc48728d17f2c5e18026a172f51d429 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 6 Sep 2024 15:01:43 -0700 Subject: [PATCH 1/4] bump version to 0.7.3 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 468ee72c..fc47a85f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2099,7 +2099,7 @@ dependencies = [ [[package]] name = "kit" -version = "0.7.2" +version = "0.7.3" dependencies = [ "alloy", "alloy-sol-macro", diff --git a/Cargo.toml b/Cargo.toml index 2b30fa56..00670257 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kit" -version = "0.7.2" +version = "0.7.3" edition = "2021" [build-dependencies] From 135b1b60bd078b3e02f51b608684371d85d191a2 Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Fri, 6 Sep 2024 15:02:29 -0700 Subject: [PATCH 2/4] run-tests: fix relative paths (they are now wrt the file) --- src/run_tests/mod.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index cab204fe..a85bd26a 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -25,11 +25,17 @@ pub mod types; use types::*; impl Config { - fn expand_home_paths(mut self: Config) -> Config { + fn expand_home_paths(mut self: Config, config_path: &Path) -> Config { + let config_path = config_path.parent().unwrap(); self.runtime = match self.runtime { Runtime::FetchVersion(version) => Runtime::FetchVersion(version), Runtime::RepoPath(runtime_path) => { - Runtime::RepoPath(expand_home_path(&runtime_path).unwrap_or(runtime_path)) + Runtime::RepoPath(expand_home_path(&runtime_path) + .unwrap_or_else(|| { + fs::canonicalize(config_path.join(&runtime_path)) + .unwrap_or_else(|_| runtime_path) + }) + ) } }; for test in self.tests.iter_mut() { @@ -39,7 +45,11 @@ impl Config { .map(|p| expand_home_path(&p).unwrap_or_else(|| p.clone())) .collect(); for node in test.nodes.iter_mut() { - node.home = expand_home_path(&node.home).unwrap_or_else(|| node.home.clone()); + node.home = expand_home_path(&node.home) + .unwrap_or_else(|| { + fs::canonicalize(config_path.join(&node.home)) + .unwrap_or_else(|_| node.home.clone()) + }); } } self @@ -82,8 +92,8 @@ fn load_config(config_path: &Path) -> Result<(PathBuf, Config)> { let content = fs::read_to_string(&config_path)?; Ok(( - config_path, - toml::from_str::(&content)?.expand_home_paths(), + config_path.clone(), + toml::from_str::(&content)?.expand_home_paths(&config_path), )) } @@ -757,6 +767,7 @@ pub async fn execute(config_path: PathBuf) -> Result<()> { let (config_path, config) = load_config(&config_path)?; + println!("{:?}", std::env::current_dir()); debug!("{:?}", config); // TODO: factor out with boot_fake_node? From b0504b4c1de10257a8cb22a1c4b9a90e6008faca Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Mon, 9 Sep 2024 21:30:50 -0700 Subject: [PATCH 3/4] fix tests --- Cargo.lock | 4 +-- Cargo.toml | 2 +- src/build/mod.rs | 16 +++++++-- src/run_tests/mod.rs | 5 ++- src/view_api/mod.rs | 81 +++++++++++++++++++++++++++++++++++--------- 5 files changed, 85 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc47a85f..d2cfb1e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2077,8 +2077,8 @@ dependencies = [ [[package]] name = "kinode_process_lib" -version = "0.9.0" -source = "git+https://github.com/kinode-dao/process_lib.git?tag=v0.9.0#284f202376b3cd3ce0c03aa660a006fc6187f236" +version = "0.9.2" +source = "git+https://github.com/kinode-dao/process_lib.git?rev=9ac9e51#9ac9e513c0228f2dcfe8999ed4ca2c38246ee3db" dependencies = [ "alloy", "alloy-primitives", diff --git a/Cargo.toml b/Cargo.toml index 00670257..5e6c6143 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ color-eyre = { version = "0.6", features = ["capture-spantrace"] } dirs = "5.0" fs-err = "2.11" hex = "0.4" -kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib.git", tag = "v0.9.0" } +kinode_process_lib = { git = "https://github.com/kinode-dao/process_lib.git", rev = "9ac9e51" } nix = { version = "0.27", features = ["process", "signal", "term"] } regex = "1" reqwest = { version = "0.12", features = ["json"] } diff --git a/src/build/mod.rs b/src/build/mod.rs index 86c8710a..d9da5a2a 100644 --- a/src/build/mod.rs +++ b/src/build/mod.rs @@ -886,15 +886,21 @@ async fn fetch_dependencies( true, )).await { debug!("Failed to build self as dependency: {e:?}"); - } else if let Err(e) = fetch_local_built_dependency( + } else if let Err(e) = fetch_local_built_dependency( apis, wasm_paths, package_dir, ) { debug!("Failed to fetch self as dependency: {e:?}"); }; + let canon_package_dir = package_dir.canonicalize()?; for local_dependency in &local_dependencies { // build dependency + let local_dep_deps = local_dependencies + .clone() + .into_iter() + .filter(|d| *d != canon_package_dir) + .collect(); Box::pin(execute( local_dependency, true, @@ -904,7 +910,7 @@ async fn fetch_dependencies( url.clone(), download_from, default_world, - vec![], // TODO: what about deps-of-deps? + local_dep_deps, vec![], force, verbose, @@ -1246,7 +1252,11 @@ async fn compile_package( // zip & place API inside of pkg/ to publish API if target_api_dir.exists() { - for path in add_paths_to_api { + let mut api_includes = add_paths_to_api.clone(); + if let Some(ref metadata_includes) = metadata.properties.api_includes { + api_includes.extend_from_slice(metadata_includes); + } + for path in api_includes { let path = if path.exists() { path } else { diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index a85bd26a..0c34c330 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -325,7 +325,7 @@ async fn build_packages( let SetupCleanupReturn { send_to_cleanup, send_to_kill, - task_handles: _, + task_handles, cleanup_context: _cleanup_context, mut master_node_port, node_cleanup_infos, @@ -418,6 +418,9 @@ async fn build_packages( info!("Cleaning up node to host dependencies."); let _ = send_to_cleanup.send(false); + for handle in task_handles { + handle.await.unwrap(); + } Ok((setup_packages, test_package_paths)) } diff --git a/src/view_api/mod.rs b/src/view_api/mod.rs index e2105bb5..8c78a8fa 100644 --- a/src/view_api/mod.rs +++ b/src/view_api/mod.rs @@ -3,14 +3,18 @@ use std::path::PathBuf; use color_eyre::{eyre::eyre, Result, Section}; use fs_err as fs; use serde_json::json; -use tracing::{info, instrument}; +use tracing::{info, instrument, warn}; use crate::{boot_fake_node::extract_zip, inject_message, KIT_CACHE, KIT_LOG_PATH_DEFAULT}; #[instrument(level = "trace", skip_all)] -fn make_app_store_message(node: Option<&str>, message: &serde_json::Value) -> Result { +fn make_app_store_message( + process_name: &str, + node: Option<&str>, + message: &serde_json::Value, +) -> Result { inject_message::make_message( - "main:app_store:sys", + &format!("{process_name}:app_store:sys"), Some(5), &message.to_string(), node, @@ -22,7 +26,22 @@ fn make_app_store_message(node: Option<&str>, message: &serde_json::Value) -> Re #[instrument(level = "trace", skip_all)] fn make_list_apis(node: Option<&str>) -> Result { let message = json!("Apis"); - make_app_store_message(node, &message) + make_app_store_message("main", node, &message) +} + +#[instrument(level = "trace", skip_all)] +fn make_get_app( + node: Option<&str>, + package_name: &str, + publisher_node: &str, +) -> Result { + let message = json!({ + "GetApp": { + "package_name": package_name, + "publisher_node": publisher_node, + }, + }); + make_app_store_message("chain", node, &message) } #[instrument(level = "trace", skip_all)] @@ -37,7 +56,7 @@ fn make_get_api( "publisher_node": publisher_node, }, }); - make_app_store_message(node, &message) + make_app_store_message("main", node, &message) } #[instrument(level = "trace", skip_all)] @@ -46,22 +65,20 @@ fn make_download( package_name: &str, publisher_node: &str, download_from: Option<&str>, - desired_version_hash: Option<&str>, + desired_version_hash: &str, ) -> Result { let download_from = download_from.unwrap_or_else(|| publisher_node); let message = json!({ - "Download": { + "LocalDownload": { "package_id": { "package_name": package_name, "publisher_node": publisher_node, }, "download_from": download_from, - "mirror": false, - "auto_update": false, "desired_version_hash": desired_version_hash, }, }); - make_app_store_message(node, &message) + make_app_store_message("downloads", node, &message) } #[instrument(level = "trace", skip_all)] @@ -77,7 +94,35 @@ fn split_package_id(package_id: &str) -> Result<(String, String)> { } #[instrument(level = "trace", skip_all)] -async fn parse_response(response: reqwest::Response) -> Result<(String, Option>)> { +async fn get_version_hash( + node: Option<&str>, + url: &str, + package_name: &str, + publisher_node: &str, +) -> Result { + let request = make_get_app(node, package_name, publisher_node)?; + let response = inject_message::send_request(url, request).await?; + let (body, _) = parse_response(response, url).await?; + let body: serde_json::Value = serde_json::from_str(&body)?; + let Some(result) = body.get("GetApp") else { + return Err(eyre!("Couldn't get version hash: bad response from node at {url}: {body}")); + }; + return match result { + serde_json::Value::String(s) => Ok(s.clone()), + serde_json::Value::Null => { + warn!("Couldn't get version hash: got Null from node at {url}: {body}"); + Ok(String::new()) + } + _ => { + Err(eyre!( + "Couldn't get version hash: got unexpected result from node at {url}: {body}" + )) + } + }; +} + +#[instrument(level = "trace", skip_all)] +async fn parse_response(response: reqwest::Response, url: &str) -> Result<(String, Option>)> { let inject_message::Response { body, lazy_load_blob, .. } = inject_message::parse_response(response) .await @@ -85,7 +130,7 @@ async fn parse_response(response: reqwest::Response) -> Result<(String, Option, ) -> Result<()> { let (package_name, publisher_node) = split_package_id(package_id)?; + let desired_version_hash = match desired_version_hash { + Some(hash) => hash.to_string(), + None => get_version_hash(node, url, &package_name, &publisher_node).await?, + }; let request = make_download( node, &package_name, &publisher_node, download_from, - desired_version_hash, + &desired_version_hash, )?; let response = inject_message::send_request(url, request).await?; - let (body, _) = parse_response(response).await?; + let (body, _) = parse_response(response, url).await?; if body.contains("Success") { Ok(()) } else if body.contains("Started") { @@ -177,7 +226,7 @@ async fn download( async fn list_apis(node: Option<&str>, url: &str, verbose: bool) -> Result { let request = make_list_apis(node)?; let response = inject_message::send_request(url, request).await?; - let (body, _) = parse_response(response).await?; + let (body, _) = parse_response(response, url).await?; let body = serde_json::from_str::(&body)?; let body = rewrite_list_apis(body)?; if verbose { @@ -198,7 +247,7 @@ async fn get_api( let (package_name, publisher_node) = split_package_id(package_id)?; let request = make_get_api(node, &package_name, &publisher_node)?; let response = inject_message::send_request(url, request).await?; - let (body, blob) = parse_response(response).await?; + let (body, blob) = parse_response(response, url).await?; let zip_dir = if let Some(blob) = blob { // get_api success let api_name = format!("{}-api", package_id); From 0bc3b4c15b1945b30f2fc051b4b287c70dd8c11c Mon Sep 17 00:00:00 2001 From: hosted-fornet Date: Wed, 11 Sep 2024 13:40:45 -0700 Subject: [PATCH 4/4] run-tests: generalize scripts --- src/run_tests/mod.rs | 38 ++++++++++++++++++++++++++------------ src/run_tests/types.rs | 10 ++-------- src/setup/mod.rs | 9 +++++++-- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/src/run_tests/mod.rs b/src/run_tests/mod.rs index 0c34c330..3c57293b 100644 --- a/src/run_tests/mod.rs +++ b/src/run_tests/mod.rs @@ -337,7 +337,6 @@ async fn build_packages( let anvil_process = chain::start_chain(test.fakechain_router, true, recv_kill_in_start_chain, false).await?; - // Process each node boot_nodes( &nodes, &test.fakechain_router, @@ -693,11 +692,21 @@ async fn handle_test( let setup_scripts: Vec = test .setup_scripts .iter() - .map(|s| { - let p = test_dir_path.join(&s.path).canonicalize().unwrap(); - let p = p.to_str().unwrap(); + .map(|script| { + let command = script + .split_whitespace() + .map(|item| { + test_dir_path + .join(&item) + .canonicalize() + .ok() + .and_then(|p| p.to_str().map(|s| s.to_string())) + .unwrap_or_else(|| item.to_string()) + }) + .collect::>() + .join(" "); Command::new("bash") - .args(["-c", &format!("{} {}", p, &s.args)]) + .args(["-c", &command]) .spawn() .expect("") .id() as i32 @@ -741,13 +750,18 @@ async fn handle_test( .await; for script in test.test_scripts { - let p = test_dir_path.join(&script.path).canonicalize().unwrap(); - let p = p.to_str().unwrap(); - let command = if script.args.is_empty() { - p.to_string() - } else { - format!("{} {}", p, script.args) - }; + let command = script + .split_whitespace() + .map(|item| { + test_dir_path + .join(&item) + .canonicalize() + .ok() + .and_then(|p| p.to_str().map(|s| s.to_string())) + .unwrap_or_else(|| item.to_string()) + }) + .collect::>() + .join(" "); build::run_command(Command::new("bash").args(["-c", &command]), false)?; } diff --git a/src/run_tests/types.rs b/src/run_tests/types.rs index 20de8542..7c4fa8ab 100644 --- a/src/run_tests/types.rs +++ b/src/run_tests/types.rs @@ -25,9 +25,9 @@ pub enum Runtime { pub struct Test { pub dependency_package_paths: Vec, pub setup_packages: Vec, - pub setup_scripts: Vec