file transfer worker lib

src/build/mod.rs

@@ -12,7 +12,7 @@ use color_eyre::{
 };
 use fs_err as fs;
 use serde::{Deserialize, Serialize};
-use tracing::{info, instrument, warn};
+use tracing::{debug, info, instrument, warn};
 
 use kinode_process_lib::{PackageId, kernel_types::Erc721Metadata};
 
@@ -586,8 +586,10 @@ async fn compile_package_and_ui(
     default_world: Option<&str>,
     download_from: Option<&str>,
     local_dependencies: Vec<PathBuf>,
+    add_paths_to_api: Vec<PathBuf>,
     force: bool,
     verbose: bool,
+    ignore_deps: bool,
 ) -> Result<()> {
     compile_and_copy_ui(package_dir, valid_node, verbose).await?;
     compile_package(
@@ -598,8 +600,10 @@ async fn compile_package_and_ui(
         default_world,
         download_from,
         local_dependencies,
+        add_paths_to_api,
         force,
         verbose,
+        ignore_deps,
     )
     .await?;
     Ok(())
@@ -664,19 +668,73 @@ async fn compile_package_item(
     Ok(())
 }
 
+#[instrument(level = "trace", skip_all)]
+fn fetch_local_built_dependency(
+    apis: &mut HashMap<String, Vec<u8>>,
+    wasm_paths: &mut HashSet<PathBuf>,
+    local_dependency: &Path,
+) -> Result<()> {
+    for entry in local_dependency.join("api").read_dir()? {
+        let entry = entry?;
+        let path = entry.path();
+        let maybe_ext = path.extension().and_then(|s| s.to_str());
+        if Some("wit") == maybe_ext {
+            let file_name = path
+                .file_name()
+                .and_then(|s| s.to_str())
+                .unwrap_or_default();
+            let wit_contents = fs::read(&path)?;
+            apis.insert(file_name.into(), wit_contents);
+        }
+    }
+    for entry in local_dependency.join("target").join("api").read_dir()? {
+        let entry = entry?;
+        let path = entry.path();
+        let maybe_ext = path.extension().and_then(|s| s.to_str());
+        if Some("wasm") == maybe_ext {
+            wasm_paths.insert(path);
+        }
+    }
+    Ok(())
+}
+
 #[instrument(level = "trace", skip_all)]
 async fn fetch_dependencies(
+    package_dir: &Path,
     dependencies: &Vec<String>,
     apis: &mut HashMap<String, Vec<u8>>,
     wasm_paths: &mut HashSet<PathBuf>,
     url: Option<String>,
     download_from: Option<&str>,
-    local_dependencies: Vec<PathBuf>,
+    mut local_dependencies: Vec<PathBuf>,
     features: &str,
     default_world: Option<&str>,
     force: bool,
     verbose: bool,
 ) -> Result<()> {
+    if let Err(e) = Box::pin(execute(
+        package_dir,
+        true,
+        false,
+        true,
+        features,
+        url.clone(),
+        download_from,
+        default_world,
+        vec![],  // TODO: what about deps-of-deps?
+        vec![],
+        force,
+        verbose,
+        true,
+    )).await {
+        debug!("Failed to build self as dependency: {e:?}");
+    } else  if let Err(e) = fetch_local_built_dependency(
+        apis,
+        wasm_paths,
+        package_dir,
+    ) {
+        debug!("Failed to fetch self as dependency: {e:?}");
+    };
     for local_dependency in &local_dependencies {
         // build dependency
         Box::pin(execute(
@@ -689,34 +747,17 @@ async fn fetch_dependencies(
             download_from,
             default_world,
             vec![],  // TODO: what about deps-of-deps?
-            verbose,
+            vec![],
             force,
+            verbose,
+            false,
         )).await?;
-        for entry in local_dependency.join("api").read_dir()? {
-            let entry = entry?;
-            let path = entry.path();
-            let maybe_ext = path.extension().and_then(|s| s.to_str());
-            if Some("wit") == maybe_ext {
-                let file_name = path
-                    .file_name()
-                    .and_then(|s| s.to_str())
-                    .unwrap_or_default();
-                let wit_contents = fs::read(&path)?;
-                apis.insert(file_name.into(), wit_contents);
-            }
-        }
-        for entry in local_dependency.join("target").join("api").read_dir()? {
-            let entry = entry?;
-            let path = entry.path();
-            let maybe_ext = path.extension().and_then(|s| s.to_str());
-            if Some("wasm") == maybe_ext {
-                wasm_paths.insert(path);
-            }
-        }
+        fetch_local_built_dependency(apis, wasm_paths, &local_dependency)?;
     }
     let Some(ref url) = url else {
         return Ok(());
     };
+    local_dependencies.push(package_dir.into());
     let local_dependencies: HashSet<&str> = local_dependencies
         .iter()
         .map(|p| p.file_name().and_then(|f| f.to_str()).unwrap())
@@ -832,7 +873,11 @@ fn get_imports_exports_from_wasm(
 fn find_non_standard(
     package_dir: &Path,
     wasm_paths: HashSet<PathBuf>,
-) -> Result<(HashMap<String, Vec<PathBuf>>, HashMap<String, PathBuf>)> {
+) -> Result<(
+    HashMap<String, Vec<PathBuf>>,
+    HashMap<String, PathBuf>,
+    HashSet<PathBuf>,
+)> {
     let mut imports = HashMap::new();
     let mut exports = HashMap::new();
 
@@ -844,11 +889,15 @@ fn find_non_standard(
         }
         get_imports_exports_from_wasm(&path, &mut imports, &mut exports, true)?;
     }
-    for wasm_path in wasm_paths {
-        get_imports_exports_from_wasm(&wasm_path, &mut imports, &mut exports, false)?;
+    for wasm_path in &wasm_paths {
+        get_imports_exports_from_wasm(wasm_path, &mut imports, &mut exports, false)?;
     }
 
-    Ok((imports, exports))
+    let others = wasm_paths
+        .difference(&exports.values().map(|p| p.clone()).collect())
+        .map(|p| p.clone())
+        .collect();
+    Ok((imports, exports, others))
 }
 
 /// package dir looks like:
@@ -882,15 +931,18 @@ async fn compile_package(
     default_world: Option<&str>,
     download_from: Option<&str>,
     local_dependencies: Vec<PathBuf>,
+    add_paths_to_api: Vec<PathBuf>,
     force: bool,
     verbose: bool,
+    ignore_deps: bool,
 ) -> Result<()> {
     let metadata = read_metadata(package_dir)?;
     let mut checked_rust = false;
     let mut checked_py = false;
     let mut checked_js = false;
     let mut apis = HashMap::new();
     let mut wasm_paths = HashSet::new();
+    let mut dependencies = HashSet::new();
     for entry in package_dir.read_dir()? {
         let entry = entry?;
         let path = entry.path();
@@ -933,31 +985,29 @@ async fn compile_package(
                 }
 
                 // fetch dependency apis: to be used in build
-                if let Some(ref dependencies) = metadata.properties.dependencies {
-                    if dependencies.is_empty() {
-                        continue;
-                    }
-                    //let Some(ref url) = url else {
-                    //    // TODO: can we use kit-cached deps?
-                    //    return Err(eyre!("Need a node to be able to fetch dependencies"));
-                    //};
-                    fetch_dependencies(
-                        dependencies,
-                        &mut apis,
-                        &mut wasm_paths,
-                        url.clone(),
-                        download_from,
-                        local_dependencies.clone(),
-                        features,
-                        default_world,
-                        force,
-                        verbose,
-                    ).await?;
+                if let Some(ref deps) = metadata.properties.dependencies {
+                    dependencies.extend(deps);
                 }
             }
         }
     }
 
+    if !ignore_deps && !dependencies.is_empty() {
+        fetch_dependencies(
+            package_dir,
+            &dependencies.iter().map(|s| s.to_string()).collect(),
+            &mut apis,
+            &mut wasm_paths,
+            url.clone(),
+            download_from,
+            local_dependencies.clone(),
+            features,
+            default_world,
+            force,
+            verbose,
+        ).await?;
+    }
+
     let wit_world = default_world.unwrap_or_else(|| match metadata.properties.wit_version {
         None => DEFAULT_WORLD_0_7_0,
         Some(0) | _ => DEFAULT_WORLD_0_8_0,
@@ -985,38 +1035,71 @@ async fn compile_package(
     let target_api_dir = package_dir.join("target").join("api");
     if api_dir.exists() {
         copy_dir(&api_dir, &target_api_dir)?;
+    } else if !target_api_dir.exists() {
+        fs::create_dir_all(&target_api_dir)?;
     }
 
-    // find non-standard imports/exports -> compositions
-    let (importers, exporters) = find_non_standard(package_dir, wasm_paths)?;
+    if !ignore_deps {
+        // find non-standard imports/exports -> compositions
+        let (importers, exporters, others) = find_non_standard(package_dir, wasm_paths)?;
+        println!("{importers:?} {exporters:?} {others:?}");
+
+        // compose
+        for (import, import_paths) in importers {
+            let Some(export_path) = exporters.get(&import) else {
+                return Err(eyre!(
+                    "Processes {import_paths:?} required export {import} not found in `pkg/`.",
+                ));
+            };
+            let export_path = export_path.to_str().unwrap();
+            for import_path in import_paths {
+                let import_path_str = import_path.to_str().unwrap();
+                run_command(
+                    Command::new("wasm-tools")
+                        .args([
+                            "compose",
+                            import_path_str,
+                            "-d",
+                            export_path,
+                            "-o",
+                            import_path_str,
+                        ]),
+                    false,
+                )?;
+            }
+        }
 
-    // compose
-    for (import, import_paths) in importers {
-        let Some(export_path) = exporters.get(&import) else {
-            return Err(eyre!(
-                "Processes {import_paths:?} required export {import} not found in `pkg/`.",
-            ));
-        };
-        let export_path = export_path.to_str().unwrap();
-        for import_path in import_paths {
-            let import_path_str = import_path.to_str().unwrap();
-            run_command(
-                Command::new("wasm-tools")
-                    .args([
-                        "compose",
-                        import_path_str,
-                        "-d",
-                        export_path,
-                        "-o",
-                        import_path_str,
-                    ]),
-                false,
+        // copy others into pkg/
+        for path in &others {
+            fs::copy(
+                path,
+                package_dir
+                    .join("pkg")
+                    .join(path.file_name().and_then(|f| f.to_str()).unwrap())
             )?;
         }
     }
 
     // zip & place API inside of pkg/ to publish API
     if target_api_dir.exists() {
+        for path in add_paths_to_api {
+            let path = if path.exists() {
+                path
+            } else {
+                package_dir.join(path).canonicalize().unwrap_or_default()
+            };
+            if !path.exists() {
+                warn!("Given path to add to API does not exist: {path:?}");
+                continue;
+            }
+            if let Err(e) = fs::copy(
+                &path,
+                target_api_dir.join(path.file_name().and_then(|f| f.to_str()).unwrap()),
+            ) {
+                warn!("Could not add path {path:?} to API: {e:?}");
+            }
+        }
+
         let zip_path = package_dir.join("pkg").join("api.zip");
         let zip_path = zip_path.to_str().unwrap();
         zip_directory(&target_api_dir, zip_path)?;
@@ -1036,8 +1119,10 @@ pub async fn execute(
     download_from: Option<&str>,
     default_world: Option<&str>,
     local_dependencies: Vec<PathBuf>,
+    add_paths_to_api: Vec<PathBuf>,
     force: bool,
     verbose: bool,
+    ignore_deps: bool,  // for internal use; may cause problems when adding recursive deps
 ) -> Result<()> {
     if !package_dir.join("pkg").exists() {
         if Some(".DS_Store") == package_dir.file_name().and_then(|s| s.to_str()) {
@@ -1093,8 +1178,10 @@ pub async fn execute(
                 default_world.clone(),
                 download_from,
                 local_dependencies,
+                add_paths_to_api,
                 force,
                 verbose,
+                ignore_deps,
             )
             .await
         }
@@ -1108,8 +1195,10 @@ pub async fn execute(
                 default_world,
                 download_from,
                 local_dependencies,
+                add_paths_to_api,
                 force,
                 verbose,
+                ignore_deps,
             )
             .await;
         }
@@ -1130,8 +1219,10 @@ pub async fn execute(
                 default_world,
                 download_from,
                 local_dependencies,
+                add_paths_to_api,
                 force,
                 verbose,
+                ignore_deps,
             )
             .await
         }