From b286ea091ffebc76e8c230982f06b12bc602fef8 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 6 Oct 2023 15:13:44 -0400 Subject: [PATCH 1/2] chore: flaky test updates (#359) --- Cargo.lock | 10 +++++ homestar-runtime/Cargo.toml | 1 + homestar-runtime/tests/cli.rs | 40 +++++++++++++------ .../metrics_node/config/settings.toml | 12 ++++++ .../network_node1/config/settings.toml | 16 ++++++++ .../network_node2/config/settings.toml | 16 ++++++++ .../fixtures/test-workflow-add-one.json | 0 .../{ => tests}/fixtures/test_v4.toml | 0 .../tests/fixtures/test_v4_alt.toml | 10 +++++ homestar-runtime/tests/fixtures/test_v6.toml | 10 +++++ homestar-runtime/tests/metrics.rs | 24 +++++++---- .../tests/test_node1/config/settings.toml | 13 ------ .../tests/test_node2/config/settings.toml | 12 ------ homestar-runtime/tests/utils.rs | 3 +- 14 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 homestar-runtime/tests/fixtures/metrics_node/config/settings.toml create mode 100644 homestar-runtime/tests/fixtures/network_node1/config/settings.toml create mode 100644 homestar-runtime/tests/fixtures/network_node2/config/settings.toml rename homestar-runtime/{ => tests}/fixtures/test-workflow-add-one.json (100%) rename homestar-runtime/{ => tests}/fixtures/test_v4.toml (100%) create mode 100644 homestar-runtime/tests/fixtures/test_v4_alt.toml create mode 100644 homestar-runtime/tests/fixtures/test_v6.toml delete mode 100644 homestar-runtime/tests/test_node1/config/settings.toml delete mode 100644 homestar-runtime/tests/test_node2/config/settings.toml diff --git a/Cargo.lock b/Cargo.lock index 886af4cd..d35ec5af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2383,6 +2383,7 @@ dependencies = [ "rand", "reqwest", "retry", + "rm_rf", "sec1", "semver", "serde", @@ -5043,6 +5044,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "rm_rf" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3443b7a35aa12ed2e99edfc0ecbefe6a53b4848305cc83e29981dfa1aea1f71e" +dependencies = [ + "stacker", +] + [[package]] name = "rmp" version = "0.8.12" diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml index 7fd4c12b..775eafda 100644 --- a/homestar-runtime/Cargo.toml +++ b/homestar-runtime/Cargo.toml @@ -179,6 +179,7 @@ predicates = { version = "3.0", default-features = false } prometheus-parse = "0.2.4" rand = { workspace = true } retry = { version = "2.0", default-features = false } +rm_rf = "0.6" serial_test = { version = "2.0", default-features = false, features = [ "file_locks", ] } diff --git a/homestar-runtime/tests/cli.rs b/homestar-runtime/tests/cli.rs index aaba16a9..dc0c6c6b 100644 --- a/homestar-runtime/tests/cli.rs +++ b/homestar-runtime/tests/cli.rs @@ -135,14 +135,16 @@ fn test_server_serial() -> Result<()> { let mut homestar_proc = Command::new(BIN.as_os_str()) .arg("start") + .arg("-c") + .arg("tests/fixtures/test_v6.toml") .arg("--db") .arg("homestar.db") .stdout(Stdio::piped()) .spawn() .unwrap(); - let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9998); + let result = retry(Fixed::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -153,6 +155,10 @@ fn test_server_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("ping") + .arg("--host") + .arg("::1") + .arg("-p") + .arg("9998") .assert() .success() .stdout(predicate::str::contains("::1")) @@ -160,6 +166,8 @@ fn test_server_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("ping") + .arg("--host") + .arg("::1") .arg("-p") .arg("9999") .assert() @@ -204,7 +212,7 @@ fn test_workflow_run_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(30), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -216,7 +224,7 @@ fn test_workflow_run_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("run") .arg("-w") - .arg("./fixtures/test-workflow-add-one.json") + .arg("tests/fixtures/test-workflow-add-one.json") .assert() .success() .stdout(predicate::str::contains( @@ -232,7 +240,7 @@ fn test_workflow_run_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("run") .arg("-w") - .arg("./fixtures/test-workflow-add-one.json") + .arg("tests/fixtures/test-workflow-add-one.json") .assert() .success() .stdout(predicate::str::contains( @@ -272,14 +280,16 @@ fn test_daemon_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("start") + .arg("-c") + .arg("tests/fixtures/test_v4_alt.toml") .arg("-d") .env("DATABASE_URL", "homestar.db") .stdout(Stdio::piped()) .assert() .success(); - let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9997); + let result = retry(Fixed::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -289,9 +299,13 @@ fn test_daemon_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("ping") + .arg("--host") + .arg("127.0.0.1") + .arg("-p") + .arg("9997") .assert() .success() - .stdout(predicate::str::contains("::1")) + .stdout(predicate::str::contains("127.0.0.1")) .stdout(predicate::str::contains("pong")); let _ = stop_all_bins(); @@ -318,7 +332,7 @@ fn test_signal_kill_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -364,7 +378,7 @@ fn test_server_v4_serial() -> Result<()> { let mut homestar_proc = Command::new(BIN.as_os_str()) .arg("start") .arg("-c") - .arg("fixtures/test_v4.toml") + .arg("tests/fixtures/test_v4.toml") .arg("--db") .arg("homestar.db") .stdout(Stdio::piped()) @@ -372,7 +386,7 @@ fn test_server_v4_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(30), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -421,7 +435,7 @@ fn test_daemon_v4_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("start") .arg("-c") - .arg("fixtures/test_v4.toml") + .arg("tests/fixtures/test_v4.toml") .arg("-d") .env("DATABASE_URL", "homestar.db") .stdout(Stdio::piped()) @@ -429,7 +443,7 @@ fn test_daemon_v4_serial() -> Result<()> { .success(); let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999); - let result = retry(Fixed::from_millis(500), || { + let result = retry(Fixed::from_millis(1000).take(30), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); diff --git a/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml b/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml new file mode 100644 index 00000000..c8f2f4b4 --- /dev/null +++ b/homestar-runtime/tests/fixtures/metrics_node/config/settings.toml @@ -0,0 +1,12 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4004 + +[node] + +[node.network] +rpc_port = 3031 +websocket_port = 9090 + +[node.network.keypair_config] +existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" } diff --git a/homestar-runtime/tests/fixtures/network_node1/config/settings.toml b/homestar-runtime/tests/fixtures/network_node1/config/settings.toml new file mode 100644 index 00000000..33090934 --- /dev/null +++ b/homestar-runtime/tests/fixtures/network_node1/config/settings.toml @@ -0,0 +1,16 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4002 + +[node] + +[node.network] +rpc_port = 3032 +websocket_port = 9092 +listen_address = "/ip4/127.0.0.1/tcp/7000" +node_addresses = [ + "/ip4/127.0.0.1/tcp/7001/p2p/16Uiu2HAm3g9AomQNeEctL2hPwLapap7AtPSNt8ZrBny4rLx1W5Dc", +] + +[node.network.keypair_config] +existing = { key_type = "ed25519", path = "./fixtures/__testkey_ed25519.pem" } diff --git a/homestar-runtime/tests/fixtures/network_node2/config/settings.toml b/homestar-runtime/tests/fixtures/network_node2/config/settings.toml new file mode 100644 index 00000000..b1cf1e0a --- /dev/null +++ b/homestar-runtime/tests/fixtures/network_node2/config/settings.toml @@ -0,0 +1,16 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4003 + +[node] + +[node.network] +rpc_port = 3033 +websocket_port = 9093 +listen_address = "/ip4/127.0.0.1/tcp/7001" +node_addresses = [ + "/ip4/127.0.0.1/tcp/7000/p2p/12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN", +] + +[node.network.keypair_config] +existing = { key_type = "secp256k1", path = "./fixtures/__testkey_secp256k1.der" } diff --git a/homestar-runtime/fixtures/test-workflow-add-one.json b/homestar-runtime/tests/fixtures/test-workflow-add-one.json similarity index 100% rename from homestar-runtime/fixtures/test-workflow-add-one.json rename to homestar-runtime/tests/fixtures/test-workflow-add-one.json diff --git a/homestar-runtime/fixtures/test_v4.toml b/homestar-runtime/tests/fixtures/test_v4.toml similarity index 100% rename from homestar-runtime/fixtures/test_v4.toml rename to homestar-runtime/tests/fixtures/test_v4.toml diff --git a/homestar-runtime/tests/fixtures/test_v4_alt.toml b/homestar-runtime/tests/fixtures/test_v4_alt.toml new file mode 100644 index 00000000..11a3c23f --- /dev/null +++ b/homestar-runtime/tests/fixtures/test_v4_alt.toml @@ -0,0 +1,10 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4007 + +[node] + +[node.network] +events_buffer_len = 1000 +rpc_port = 9997 +rpc_host = "127.0.0.1" diff --git a/homestar-runtime/tests/fixtures/test_v6.toml b/homestar-runtime/tests/fixtures/test_v6.toml new file mode 100644 index 00000000..2e35703f --- /dev/null +++ b/homestar-runtime/tests/fixtures/test_v6.toml @@ -0,0 +1,10 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4006 + +[node] + +[node.network] +events_buffer_len = 1000 +rpc_port = 9998 +rpc_host = "::1" diff --git a/homestar-runtime/tests/metrics.rs b/homestar-runtime/tests/metrics.rs index 7c70209f..889b5dc1 100644 --- a/homestar-runtime/tests/metrics.rs +++ b/homestar-runtime/tests/metrics.rs @@ -5,6 +5,7 @@ use reqwest::StatusCode; use retry::{delay::Fixed, retry, OperationResult}; use serial_test::file_serial; use std::{ + net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream}, path::PathBuf, process::{Command, Stdio}, time::Duration, @@ -12,14 +13,14 @@ use std::{ use wait_timeout::ChildExt; static BIN: Lazy = Lazy::new(|| assert_cmd::cargo::cargo_bin(BIN_NAME)); -const METRICS_URL: &str = "http://localhost:4000"; +const METRICS_URL: &str = "http://localhost:4004"; #[test] #[file_serial] fn test_metrics_serial() -> Result<()> { fn sample_metrics() -> prometheus_parse::Value { let body = retry( - Fixed::from_millis(500).take(2), + Fixed::from_millis(1000).take(10), || match reqwest::blocking::get(METRICS_URL) { Ok(response) => match response.status() { StatusCode::OK => OperationResult::Ok(response.text()), @@ -28,10 +29,9 @@ fn test_metrics_serial() -> Result<()> { Err(_) => OperationResult::Retry("Metrics server not available"), }, ) - .unwrap() - .expect("Metrics server failed to serve metrics"); + .unwrap(); - let lines: Vec<_> = body.lines().map(|s| Ok(s.to_owned())).collect(); + let lines: Vec<_> = body.unwrap().lines().map(|s| Ok(s.to_owned())).collect(); let metrics = prometheus_parse::Scrape::parse(lines.into_iter()).expect("Unable to parse metrics"); @@ -49,16 +49,26 @@ fn test_metrics_serial() -> Result<()> { let mut homestar_proc = Command::new(BIN.as_os_str()) .arg("start") .arg("-c") - .arg("tests/test_node2/config/settings.toml") + .arg("tests/fixtures/metrics_node/config/settings.toml") .arg("--db") .arg("homestar.db") .stdout(Stdio::piped()) .spawn() .unwrap(); + let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4004); + let result = retry(Fixed::from_millis(1000).take(10), || { + TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) + }); + + if result.is_err() { + homestar_proc.kill().unwrap(); + panic!("Homestar server/runtime failed to start in time"); + } + let sample1 = sample_metrics(); - let sample2 = retry(Fixed::from_millis(500).take(5), || { + let sample2 = retry(Fixed::from_millis(500).take(3), || { let sample2 = sample_metrics(); if sample1 != sample2 { OperationResult::Ok(sample2) diff --git a/homestar-runtime/tests/test_node1/config/settings.toml b/homestar-runtime/tests/test_node1/config/settings.toml deleted file mode 100644 index 305694c9..00000000 --- a/homestar-runtime/tests/test_node1/config/settings.toml +++ /dev/null @@ -1,13 +0,0 @@ -[monitoring] -process_collector_interval = 500 -metrics_port = 4000 - -[node] - -[node.network] -rpc_port = 3031 -websocket_port = 9091 -node_addresses = ["/ip4/127.0.0.1/tcp/9092/ws"] - -[node.network.keypair_config] -existing = { key_type = "ed25519", path = "../../fixtures/__testkey_ed25519.pem" } diff --git a/homestar-runtime/tests/test_node2/config/settings.toml b/homestar-runtime/tests/test_node2/config/settings.toml deleted file mode 100644 index 92570f77..00000000 --- a/homestar-runtime/tests/test_node2/config/settings.toml +++ /dev/null @@ -1,12 +0,0 @@ -[monitoring] -process_collector_interval = 500 -metrics_port = 4000 - -[node] - -[node.network] -rpc_port = 3032 -websocket_port = 9092 -node_addresses = [ - "/ip4/127.0.0.1/tcp/9091/ws/p2p/12D3KooWDpJ7As7BWAwRMfu1VU2WCqNjvq387JEYKDBj4kx6nXTN", -] diff --git a/homestar-runtime/tests/utils.rs b/homestar-runtime/tests/utils.rs index 43d94452..afedace7 100644 --- a/homestar-runtime/tests/utils.rs +++ b/homestar-runtime/tests/utils.rs @@ -67,6 +67,7 @@ pub(crate) fn stop_ipfs() -> Result<()> { .stderr(Stdio::null()) .status() .context("failed to stop IPFS daemon")?; + rm_rf::ensure_removed(path).unwrap(); Ok(()) } @@ -97,7 +98,7 @@ pub(crate) fn kill_homestar_process() -> Result<()> { ); let _result = signal::kill(Pid::from_raw(pid.try_into().unwrap()), Signal::SIGTERM); - let _result = retry(Fixed::from_millis(500), || { + let _result = retry(Fixed::from_millis(1000).take(5), || { Command::new(BIN.as_os_str()) .arg("ping") .assert() From 0eeb0bfb11c7d8a9c72ba4fcea445218ae8d3d30 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Mon, 9 Oct 2023 14:50:08 -0400 Subject: [PATCH 2/2] feat: builds, packages, pushing contains to ghcr (#360) Closes https://github.com/ipvm-wg/homestar/issues/276. We'll open a new issue for more specific packages: brew, arch-linux, wix/windows. This will need some further testing to be absolutely sure, cc @walkah. Includes: - Fixes bug with empty check on annouce_addresses - local cross-rs setup for builds - github action workflows: builds.yml tied to release * builds binaries (cross if need be) for aarch64-unknown-linux-gnu, aarch64-unknown-linux-musl, aarch64-apple-darwin, x86_64-unknown-linux-gnu, x86_64-unknown-linux-musl, x86_64-apple-darwin, x86_64-pc-windows-msvc, and x86_64-unknown-freebsd (for now) * builds packages for .deb, .rpm (sets out todos for brew, wix, aur) * docker push to ghcr.io and tag mult. versions --- .github/workflows/builds.yml | 188 ++++++++++++++++++ .github/workflows/coverage.yml | 7 +- .github/workflows/docker.yml | 2 +- Cross.toml | 19 ++ README.md | 2 +- docker/Dockerfile | 3 +- flake.lock | 18 +- flake.nix | 12 +- homestar-runtime/Cargo.toml | 38 +++- homestar-runtime/src/network/swarm.rs | 2 +- homestar-runtime/tests/cli.rs | 30 +-- homestar-runtime/tests/fixtures/test_v4.toml | 4 + .../tests/fixtures/test_v4_alt.toml | 2 +- homestar-runtime/tests/fixtures/test_v6.toml | 2 +- .../tests/fixtures/test_workflow.toml | 9 + homestar-runtime/tests/metrics.rs | 18 +- 16 files changed, 317 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/builds.yml create mode 100644 Cross.toml create mode 100644 homestar-runtime/tests/fixtures/test_workflow.toml diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml new file mode 100644 index 00000000..b30ceff2 --- /dev/null +++ b/.github/workflows/builds.yml @@ -0,0 +1,188 @@ +name: ⚃ Builds + +on: + workflow_dispatch: + inputs: + force-builds: + required: true + type: boolean + description: Publish Builds at Anytime + + release: + types: [published] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + binary-builds: + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-unknown-linux-gnu + - target: aarch64-unknown-linux-musl + - target: aarch64-apple-darwin + os: macos-latest + - target: x86_64-unknown-linux-gnu + - target: x86_64-unknown-linux-musl + - target: x86_64-apple-darwin + os: macos-latest + - target: x86_64-pc-windows-msvc + os: windows-latest + - target: x86_64-unknown-freebsd + + permissions: + contents: write + + timeout-minutes: 60 + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install musl-tools + run: sudo apt update && sudo apt install -y musl-dev musl-tools + if: matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' + + - name: Cache Project + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + shared-key: check-${{ matrix.target }}-${{ matrix.os }} + + - name: Install Rust Toolchain + id: toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Override rust-toolchain.toml + run: rustup override set ${{steps.toolchain.outputs.name}} + + - name: cross-build + uses: taiki-e/setup-cross-toolchain-action@v1 + with: + target: ${{ matrix.target }} + + - name: crt-static + run: echo "RUSTFLAGS=${RUSTFLAGS} -C target-feature=+crt-static" >>"${GITHUB_ENV}" + if: endsWith(matrix.target, 'windows-msvc') + + - name: Publish Binary in Release + uses: taiki-e/upload-rust-binary-action@v1 + with: + bin: homestar + target: ${{ matrix.target }} + tar: all + zip: windows + include: LICENSE,README.md + token: ${{ secrets.GITHUB_TOKEN }} + + build-packages: + runs-on: ubuntu-latest + + env: + LINUX_TARGET: x86_64-unknown-linux-musl + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Cache Project + uses: Swatinem/rust-cache@v2 + with: + cache-on-failure: true + shared-key: check-${{ env.LINUX_TARGET }}-ubuntu-latest + + - name: Install musl-tools + run: sudo apt update && sudo apt install -y musl-dev musl-tools + + - name: Install cargo-deb + uses: taiki-e/cache-cargo-install-action@v1 + with: + tool: cargo-deb + + - name: Install cargo-generate-rpm + uses: taiki-e/cache-cargo-install-action@v1 + with: + tool: cargo-generate-rpm + + - name: Create .deb + run: cargo deb -p homestar-runtime --target ${{env.LINUX_TARGET}} + + - name: Create .rpm + run: cargo generate-rpm + + - name: Publish Release Artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.target }} + path: | + target/generate-rpm/ + target/${{env.LINUX_TARGET}}/debian/ + if-no-files-found: error + + - name: Publish Package in Release + uses: softprops/action-gh-release@v1 + if: startsWith(github.ref, 'refs/tags/') + with: + files: | + target/${{env.LINUX_TARGET}}/debian/*.deb + target/generate-rpm/*.rpm + + # TODO: brew formula (Macs), cargo-wix (Windows Installs), cargo-aur (Arch) + + docker-build: + runs-on: ubuntu-latest + + env: + DOCKER_BUILDKIT: '1' + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + + permissions: + contents: read + id-token: write + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Setup Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: "--debug" + + - name: Metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern=v{{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest + type=sha + + - name: Docker Build & Push + uses: docker/build-push-action@v5 + with: + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + cache-to: type=inline + file: docker/Dockerfile + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 285acf43..5658f422 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -75,8 +75,11 @@ jobs: - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - - name: Generate Code Coverage - run: cargo llvm-cov --all-features --workspace --doctests --lcov --output-path lcov.info + - name: Install cargo-nextest + uses: taiki-e/install-action@nextest + + - name: Generate code coverage (nextest) + run: cargo llvm-cov nextest --all-features --workspace --profile=ci --lcov --output-path lcov.info - name: Upload to codecov.io uses: codecov/codecov-action@v3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b6a3175b..3c02e12c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -32,7 +32,7 @@ jobs: cache-to: type=gha,mode=max file: docker/Dockerfile context: . - platforms: linux/amd64 + platforms: linux/amd64,linux/arm64 push: false tags: | ${{ github.repository_owner }}/homestar:latest diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 00000000..688d1ac7 --- /dev/null +++ b/Cross.toml @@ -0,0 +1,19 @@ +[build.env] +passthrough = [ + "CARGO_INCREMENTAL", + "RUST_BACKTRACE", + "CARGO_TERM_COLOR", + "RUSTFLAGS", +] + +[target.x86_64-unknown-linux-musl] +image = "burntsushi/cross:x86_64-unknown-linux-musl" + +[target.aarch64-unknown-linux-musl] +image = "burntsushi/cross:aarch64-unknown-linux-gnu" + +[target.x86_64-apple-darwin] +image = "freeznet/x86_64-apple-darwin-cross:11.3" + +[target.aarch64-apple-darwin] +image = "freeznet/aarch64-apple-darwin-cross:11.3" diff --git a/README.md b/README.md index 3a1b0674..58ea6072 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ If you're looking to help develop `homestar`, please dive right into our [development](./DEVELOPMENT.md) guide. -Otherwise, the easiest way to get started and "see" `homestar` in action is to +Otherwise, the easiest way to get started and see `homestar` in action is to follow-along and run our image-processing [websocket relay](./examples/websocket-relay) example, which integrates `homestar` with a browser application to run a diff --git a/docker/Dockerfile b/docker/Dockerfile index f0d29caa..53097d50 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,8 +40,7 @@ ENTRYPOINT ["/bin/bash"] # cache depencies RUN mkdir .cargo RUN cargo vendor > .cargo/config -RUN --mount=type=cache,target=$CARGO_HOME/registry \ - cargo install diesel_cli --no-default-features --features "sqlite-bundled" +RUN cargo install diesel_cli --no-default-features --features "sqlite-bundled" RUN diesel setup --database-url $DATABASE_URL RUN diesel migration run --migration-dir ./homestar-runtime/migrations RUN --mount=type=cache,id=cargo,target=$CARGO_HOME/registry \ diff --git a/flake.lock b/flake.lock index 0514ad76..edcb403f 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1696267196, - "narHash": "sha256-AAQ/2sD+0D18bb8hKuEEVpHUYD1GmO2Uh/taFamn6XQ=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "4f910c9827911b1ec2bf26b5a062cd09f8d89f85", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -36,11 +36,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1696234590, - "narHash": "sha256-mgOzQYTvaTT4bFopVOadlndy2RPwLy60rDjIWOGujwo=", + "lastModified": 1696577711, + "narHash": "sha256-94VRjvClIKDym1QRqPkX5LTQoAwZ1E6QE/3dWtOXSIQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f902cb49892d300ff15cb237e48aa1cad79d68c3", + "rev": "a2eb207f45e4a14a1e3019d9e3863d1e208e2295", "type": "github" }, "original": { @@ -68,11 +68,11 @@ ] }, "locked": { - "lastModified": 1696212720, - "narHash": "sha256-kuI/n1LpXv9S1wXfIzvYJudBfQeF0YGS4G8xJQQxM/M=", + "lastModified": 1696558324, + "narHash": "sha256-TnnP4LGwDB8ZGE7h2n4nA9Faee8xPkMdNcyrzJ57cbw=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "1aaa2dc3e7367f2014f939c927e9e768a0cc2f08", + "rev": "fdb37574a04df04aaa8cf7708f94a9309caebe2b", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 0069f766..1b4fe582 100644 --- a/flake.nix +++ b/flake.nix @@ -32,7 +32,14 @@ rust-toolchain = (pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml).override { extensions = ["cargo" "clippy" "rustfmt" "rust-src" "rust-std"]; - targets = ["wasm32-unknown-unknown" "wasm32-wasi"]; + targets = [ + "wasm32-unknown-unknown" + "wasm32-wasi" + "x86_64-apple-darwin" + "aarch64-apple-darwin" + "x86_64-unknown-linux-musl" + "aarch64-unknown-linux-musl" + ]; }; nightly-rustfmt = pkgs.rust-bin.nightly.latest.rustfmt; @@ -45,6 +52,8 @@ cargo-installs = with pkgs; [ cargo-deny + cargo-deb + cargo-cross cargo-expand cargo-nextest cargo-outdated @@ -53,6 +62,7 @@ cargo-unused-features cargo-udeps cargo-watch + rustup twiggy wasm-tools ]; diff --git a/homestar-runtime/Cargo.toml b/homestar-runtime/Cargo.toml index 775eafda..abbe18c1 100644 --- a/homestar-runtime/Cargo.toml +++ b/homestar-runtime/Cargo.toml @@ -23,7 +23,6 @@ autotests = false path = "src/lib.rs" bench = false doctest = true -crate-type = ["cdylib", "rlib"] [[bin]] name = "homestar" @@ -204,3 +203,40 @@ websocket-server = ["dep:axum"] all-features = true # defines the configuration attribute `docsrs` rustdoc-args = ["--cfg", "docsrs"] + +[package.metadata.deb] +maintainer = "James Walker " +license-file = ["LICENSE", "0"] +extended-description-file = "README.md" +section = "network" +priority = "optional" +assets = [ + [ + "../target/x86_64-unknown-linux-musl/release/homestar", + "usr/bin/", + "755", + ], + [ + "../CHANGELOG.md", + "usr/share/doc/homestar/", + "644", + ], + [ + "../LICENSE", + "usr/share/doc/homestar/", + "644", + ], + [ + "../README.md", + "usr/share/doc/homestar/", + "644", + ], +] + +[package.metadata.generate-rpm] +assets = [ + { source = "../target/x86_64-unknown-linux-musl/release/homestar", dest = "/usr/bin/homestar", mode = "755" }, + { source = "../CHANGELOG.md", dest = "/usr/share/doc/homestar/CHANGELOG.md", mode = "644", doc = true }, + { source = "../LICENSE", dest = "/usr/share/doc/homestar/LICENSE.md", mode = "644", doc = true }, + { source = "../README.md", dest = "/usr/share/doc/homestar/README.md", mode = "644", doc = true }, +] diff --git a/homestar-runtime/src/network/swarm.rs b/homestar-runtime/src/network/swarm.rs index 72837161..0ec8ed13 100644 --- a/homestar-runtime/src/network/swarm.rs +++ b/homestar-runtime/src/network/swarm.rs @@ -120,7 +120,7 @@ pub(crate) fn init( swarm.listen_on(settings.listen_address.to_string().parse()?)?; // add external addresses from settings - if settings.announce_addresses.is_empty() { + if !settings.announce_addresses.is_empty() { for addr in settings.announce_addresses.iter() { swarm.add_external_address(addr.clone()); } diff --git a/homestar-runtime/tests/cli.rs b/homestar-runtime/tests/cli.rs index dc0c6c6b..2fcc1b05 100644 --- a/homestar-runtime/tests/cli.rs +++ b/homestar-runtime/tests/cli.rs @@ -5,7 +5,7 @@ use anyhow::Result; use assert_cmd::prelude::*; use once_cell::sync::Lazy; use predicates::prelude::*; -use retry::{delay::Fixed, retry}; +use retry::{delay::Exponential, retry}; use serial_test::file_serial; use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr, Shutdown, SocketAddr, TcpStream}, @@ -143,8 +143,8 @@ fn test_server_serial() -> Result<()> { .spawn() .unwrap(); - let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9998); - let result = retry(Fixed::from_millis(1000).take(10), || { + let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9991); + let result = retry(Exponential::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -158,7 +158,7 @@ fn test_server_serial() -> Result<()> { .arg("--host") .arg("::1") .arg("-p") - .arg("9998") + .arg("9991") .assert() .success() .stdout(predicate::str::contains("::1")) @@ -205,14 +205,16 @@ fn test_workflow_run_serial() -> Result<()> { let mut homestar_proc = Command::new(BIN.as_os_str()) .arg("start") + .arg("-c") + .arg("tests/fixtures/test_workflow.toml") .arg("--db") .arg("homestar.db") .stdout(Stdio::piped()) .spawn() .unwrap(); - let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(1000).take(30), || { + let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 9888); + let result = retry(Exponential::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -223,6 +225,8 @@ fn test_workflow_run_serial() -> Result<()> { Command::new(BIN.as_os_str()) .arg("run") + .arg("-p") + .arg("9888") .arg("-w") .arg("tests/fixtures/test-workflow-add-one.json") .assert() @@ -239,6 +243,8 @@ fn test_workflow_run_serial() -> Result<()> { // run another one of the same! Command::new(BIN.as_os_str()) .arg("run") + .arg("-p") + .arg("9888") .arg("-w") .arg("tests/fixtures/test-workflow-add-one.json") .assert() @@ -288,8 +294,8 @@ fn test_daemon_serial() -> Result<()> { .assert() .success(); - let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9997); - let result = retry(Fixed::from_millis(1000).take(10), || { + let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9987); + let result = retry(Exponential::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -302,7 +308,7 @@ fn test_daemon_serial() -> Result<()> { .arg("--host") .arg("127.0.0.1") .arg("-p") - .arg("9997") + .arg("9987") .assert() .success() .stdout(predicate::str::contains("127.0.0.1")) @@ -332,7 +338,7 @@ fn test_signal_kill_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 3030); - let result = retry(Fixed::from_millis(1000).take(10), || { + let result = retry(Exponential::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -386,7 +392,7 @@ fn test_server_v4_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999); - let result = retry(Fixed::from_millis(1000).take(30), || { + let result = retry(Exponential::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -443,7 +449,7 @@ fn test_daemon_v4_serial() -> Result<()> { .success(); let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9999); - let result = retry(Fixed::from_millis(1000).take(30), || { + let result = retry(Exponential::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); diff --git a/homestar-runtime/tests/fixtures/test_v4.toml b/homestar-runtime/tests/fixtures/test_v4.toml index 37bdce12..f4536b6c 100644 --- a/homestar-runtime/tests/fixtures/test_v4.toml +++ b/homestar-runtime/tests/fixtures/test_v4.toml @@ -1,3 +1,7 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4010 + [node] [node.network] diff --git a/homestar-runtime/tests/fixtures/test_v4_alt.toml b/homestar-runtime/tests/fixtures/test_v4_alt.toml index 11a3c23f..f111adb9 100644 --- a/homestar-runtime/tests/fixtures/test_v4_alt.toml +++ b/homestar-runtime/tests/fixtures/test_v4_alt.toml @@ -6,5 +6,5 @@ metrics_port = 4007 [node.network] events_buffer_len = 1000 -rpc_port = 9997 +rpc_port = 9987 rpc_host = "127.0.0.1" diff --git a/homestar-runtime/tests/fixtures/test_v6.toml b/homestar-runtime/tests/fixtures/test_v6.toml index 2e35703f..938a30b2 100644 --- a/homestar-runtime/tests/fixtures/test_v6.toml +++ b/homestar-runtime/tests/fixtures/test_v6.toml @@ -6,5 +6,5 @@ metrics_port = 4006 [node.network] events_buffer_len = 1000 -rpc_port = 9998 +rpc_port = 9991 rpc_host = "::1" diff --git a/homestar-runtime/tests/fixtures/test_workflow.toml b/homestar-runtime/tests/fixtures/test_workflow.toml new file mode 100644 index 00000000..6a820d4d --- /dev/null +++ b/homestar-runtime/tests/fixtures/test_workflow.toml @@ -0,0 +1,9 @@ +[monitoring] +process_collector_interval = 500 +metrics_port = 4012 + +[node] + +[node.network] +events_buffer_len = 1000 +rpc_port = 9888 diff --git a/homestar-runtime/tests/metrics.rs b/homestar-runtime/tests/metrics.rs index 889b5dc1..9081f9de 100644 --- a/homestar-runtime/tests/metrics.rs +++ b/homestar-runtime/tests/metrics.rs @@ -2,7 +2,7 @@ use crate::utils::{stop_homestar, BIN_NAME}; use anyhow::Result; use once_cell::sync::Lazy; use reqwest::StatusCode; -use retry::{delay::Fixed, retry, OperationResult}; +use retry::{delay::Exponential, retry, OperationResult}; use serial_test::file_serial; use std::{ net::{IpAddr, Ipv4Addr, Shutdown, SocketAddr, TcpStream}, @@ -20,7 +20,7 @@ const METRICS_URL: &str = "http://localhost:4004"; fn test_metrics_serial() -> Result<()> { fn sample_metrics() -> prometheus_parse::Value { let body = retry( - Fixed::from_millis(1000).take(10), + Exponential::from_millis(500).take(20), || match reqwest::blocking::get(METRICS_URL) { Ok(response) => match response.status() { StatusCode::OK => OperationResult::Ok(response.text()), @@ -57,7 +57,7 @@ fn test_metrics_serial() -> Result<()> { .unwrap(); let socket = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 4004); - let result = retry(Fixed::from_millis(1000).take(10), || { + let result = retry(Exponential::from_millis(1000).take(10), || { TcpStream::connect(socket).map(|stream| stream.shutdown(Shutdown::Both)) }); @@ -68,17 +68,21 @@ fn test_metrics_serial() -> Result<()> { let sample1 = sample_metrics(); - let sample2 = retry(Fixed::from_millis(500).take(3), || { + let sample2 = retry(Exponential::from_millis(500).take(10), || { let sample2 = sample_metrics(); if sample1 != sample2 { OperationResult::Ok(sample2) } else { OperationResult::Retry("Samples are the same") } - }) - .unwrap(); + }); + + if sample2.is_err() { + homestar_proc.kill().unwrap(); + panic!("Could not generate a diff in sample(s)"); + } - assert_ne!(sample1, sample2); + assert_ne!(sample1, sample2.unwrap()); if let Ok(None) = homestar_proc.try_wait() { let _status_code = match homestar_proc.wait_timeout(Duration::from_secs(1)).unwrap() {