fix(ci): flaky tests

.github/workflows/nix.yml

@@ -42,7 +42,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: ./.github/actions/install-nix
-      with: 
+      with:
         cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - uses: ./.github/actions/build-nix
       with:
@@ -66,11 +66,12 @@ jobs:
           test: ./result/bin/wash --version
 
     name: wash-${{ matrix.config.target }}
-    runs-on: macos-12
+    # NOTE(vados-cosmonic): Mac have OOMed in GitHub CI on macos-12 at least once
+    runs-on: macos-12-xl
     steps:
     - uses: actions/checkout@v3
     - uses: ./.github/actions/install-nix
-      with: 
+      with:
         cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - uses: ./.github/actions/build-nix
       with:
@@ -125,7 +126,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: ./.github/actions/install-nix
-      with: 
+      with:
         cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix build -L .#checks.x86_64-linux.${{ matrix.check }}
 
@@ -134,7 +135,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: ./.github/actions/install-nix
-      with: 
+      with:
         cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix fmt
 
@@ -143,7 +144,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: ./.github/actions/install-nix
-      with: 
+      with:
         cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix run -L . -- --version
 
@@ -152,6 +153,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: ./.github/actions/install-nix
-      with: 
+      with:
         cachixAuthToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
     - run: nix develop -L --ignore-environment -c cargo pkgid

.github/workflows/rust_ci.yml

@@ -10,13 +10,14 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
-  fmt_clippy:
-    name: Fmt & Clippy Checks
+  lint:
+    name: Lint
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
-      # Cache Rust builds between runs
       - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "ubuntu-22.04-shared-cache"
       - name: Run Rust check + clippy
         run: make rust-check
 
@@ -29,11 +30,9 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3
-      # Cache Rust builds between runs
       - uses: Swatinem/rust-cache@v2
         with:
-          # Additional key to ensure OS specific caches are separated
-          key: "${{ matrix.os }}-cache"
+          shared-key: "${{ matrix.os }}-shared-cache"
       - name: Install nextest
         uses: taiki-e/install-action@nextest
       - name: Run all wash & wash-lib unit tests
@@ -44,8 +43,9 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
-      # Cache Rust builds between runs
       - uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: "ubuntu-22.04-shared-cache"
       - uses: acifani/setup-tinygo@v1
         with:
           tinygo-version: '0.27.0'

Makefile

@@ -33,8 +33,7 @@ test: ## Run unit test suite
 	@$(CARGO) nextest run $(CARGO_TEST_TARGET) --no-fail-fast -p wash-lib --features=cli
 
 test-wash-ci:
-	@$(CARGO) nextest run --profile ci --workspace --bin wash
-	@$(CARGO) nextest run --profile ci -p wash-lib --features=cli
+	@$(CARGO) nextest run --profile ci --workspace --all-features -E 'binary(wash)' -E 'package(wash-lib)'
 
 test-watch: ## Run unit tests continously, can optionally specify a target test filter.
 	@$(CARGO) watch -- $(CARGO) nextest run $(TARGET)

crates/wash-lib/src/cli/output.rs

@@ -38,28 +38,28 @@ pub struct StopCommandOutput {
 
 /// JSON output representation of the `wash link query` command
 #[derive(Debug, Deserialize)]
-pub struct LinkQueryOutput {
+pub struct LinkQueryCommandOutput {
     pub links: Vec<HashMap<String, ActorLinks>>,
     pub success: bool,
 }
 
 /// JSON output representation of the `wash get hosts` command
 #[derive(Debug, Clone, Deserialize)]
-pub struct GetHostsOutput {
+pub struct GetHostsCommandOutput {
     pub success: bool,
     pub hosts: Vec<Host>,
 }
 
 /// JSON output representation of the `wash get inventory` command
 #[derive(Debug, Clone, Deserialize)]
-pub struct GetHostInventoryOutput {
+pub struct GetHostInventoryCommandOutput {
     pub success: bool,
     pub inventory: HostInventory,
 }
 
 /// JSON output representation of the `wash get claims` command
 #[derive(Debug, Deserialize)]
-pub struct GetClaimsOutput {
+pub struct GetClaimsCommandOutput {
     pub claims: GetClaimsResponse,
     pub success: bool,
 }

flake.nix

@@ -50,6 +50,7 @@
               depsBuildBuild
               ++ optionals stdenv.hostPlatform.isDarwin [
                 darwin.apple_sdk.frameworks.CoreFoundation
+                darwin.apple_sdk.frameworks.CoreServices
                 libiconv
               ];
 

tests/common.rs

@@ -7,13 +7,19 @@ use std::{
 
 use anyhow::{bail, Context, Result};
 use rand::{distributions::Alphanumeric, Rng};
+use sysinfo::{ProcessExt, SystemExt};
 use tempfile::TempDir;
 use tokio::net::TcpStream;
-use tokio::process::Command;
+use tokio::{
+    fs::File,
+    io::AsyncWriteExt,
+    process::{Child, Command},
+    time::Duration,
+};
 
-use sysinfo::SystemExt;
-use tokio::process::Child;
+use wash_lib::cli::output::GetHostsCommandOutput;
 use wash_lib::start::{ensure_nats_server, start_nats_server, NatsConfig};
+use wasmcloud_control_interface::Host;
 
 #[allow(unused)]
 pub(crate) const LOCAL_REGISTRY: &str = "localhost:5001";
@@ -275,3 +281,152 @@ pub(crate) async fn init_actor_from_template(
     let project_dir = std::env::current_dir()?.join(actor_name);
     Ok(project_dir)
 }
+
+/// Wait until a process has a given count on the current machine
+#[allow(dead_code)]
+pub(crate) async fn wait_until_process_has_count(
+    filter: &str,
+    predicate: impl Fn(usize) -> bool,
+    timeout: Duration,
+    check_interval: Duration,
+) -> Result<()> {
+    // Check to see if process was removed
+    let mut info = sysinfo::System::new_with_specifics(
+        sysinfo::RefreshKind::new().with_processes(sysinfo::ProcessRefreshKind::new()),
+    );
+
+    tokio::time::timeout(timeout, async move {
+        loop {
+            info.refresh_processes();
+            let count = info
+                .processes()
+                .values()
+                .map(|p| p.exe().to_string_lossy())
+                .filter(|name| name.contains(filter))
+                .count();
+            if predicate(count) {
+                break;
+            };
+            tokio::time::sleep(check_interval).await;
+        }
+    })
+    .await
+    .context(format!(
+        "Failed to find find satisfactory amount of processes named [{filter}]"
+    ))?;
+
+    Ok(())
+}
+
+#[allow(dead_code)]
+pub(crate) async fn wait_for_single_host(
+    ctl_port: u16,
+    timeout: Duration,
+    check_interval: Duration,
+) -> Result<Host> {
+    tokio::time::timeout(timeout, async move {
+        loop {
+            let output = Command::new(env!("CARGO_BIN_EXE_wash"))
+                .args([
+                    "get",
+                    "hosts",
+                    "--ctl-port",
+                    ctl_port.to_string().as_str(),
+                    "--output",
+                    "json",
+                ])
+                .output()
+                .await
+                .context("get host command failed")?;
+
+            // If we fail to get hosts, then restart
+            if !output.status.success() {
+                bail!(
+                    "`wash get hosts` failed (exit code {:?}): {}",
+                    output.status.code(),
+                    String::from_utf8_lossy(&output.stdout)
+                );
+            }
+
+            let mut cmd_output: GetHostsCommandOutput = serde_json::from_slice(&output.stdout)
+                .with_context(|| {
+                    format!(
+                        "failed to parse get hosts command JSON output: {}",
+                        String::from_utf8_lossy(&output.stdout)
+                    )
+                })?;
+
+            match &cmd_output.hosts[..] {
+                [] => {}
+                [_h] => break Ok(cmd_output.hosts.remove(0)),
+                _ => bail!("unexpected received more than one host"),
+            }
+
+            tokio::time::sleep(check_interval).await;
+        }
+    })
+    .await
+    .context("failed to wait for single host to exist")?
+}
+
+/// Inits an actor build test by setting up a test directory and creating an actor from a template.
+/// Returns the paths of the test directory and actor directory.
+#[allow(dead_code)]
+pub(crate) async fn init_workspace(actor_names: Vec<&str>) -> Result<WorkspaceTestSetup> {
+    let test_dir = TempDir::new()?;
+    std::env::set_current_dir(&test_dir)?;
+
+    let project_dirs: Vec<_> =
+        futures::future::try_join_all(actor_names.iter().map(|actor_name| async {
+            let project_dir = init_actor_from_template(actor_name, "hello").await?;
+            Result::<PathBuf>::Ok(project_dir)
+        }))
+        .await?;
+
+    let members = actor_names
+        .iter()
+        .map(|actor_name| format!("\"{actor_name}\""))
+        .collect::<Vec<_>>()
+        .join(",");
+    let cargo_toml = format!(
+        "
+    [workspace]
+    members = [{members}]
+    "
+    );
+
+    let mut cargo_path = PathBuf::from(test_dir.path());
+    cargo_path.push("Cargo.toml");
+    let mut file = File::create(cargo_path).await?;
+    file.write_all(cargo_toml.as_bytes()).await?;
+    Ok(WorkspaceTestSetup {
+        test_dir,
+        project_dirs,
+    })
+}
+
+/// Wait for no hosts to be running by checking for process names,
+/// expecting that the wasmcloud process invocation contains 'beam.smp'
+#[allow(dead_code)]
+pub(crate) async fn wait_for_no_hosts() -> Result<()> {
+    wait_until_process_has_count(
+        "beam.smp",
+        |v| v == 0,
+        Duration::from_secs(10),
+        Duration::from_millis(250),
+    )
+    .await
+}
+
+/// Wait for NATS to start running by checking for process names.
+/// expecting that exactly one 'nats-server' process is running
+#[allow(dead_code)]
+pub(crate) async fn wait_for_nats_to_start() -> Result<()> {
+    wait_until_process_has_count(
+        "nats-server",
+        |v| v == 1,
+        Duration::from_secs(10),
+        Duration::from_secs(1),
+    )
+    .await
+}