diff --git a/host_core/test/host_core/benchmark/common.exs b/host_core/test/host_core/benchmark/common.exs new file mode 100644 index 00000000..0f7948dc --- /dev/null +++ b/host_core/test/host_core/benchmark/common.exs @@ -0,0 +1,52 @@ +defmodule HostCore.Benchmark.Common do + require Logger + + alias HostCore.Actors.ActorSupervisor + alias HostCore.Jetstream.Client, as: JetstreamClient + alias HostCore.Vhost.VirtualHost + alias HostCore.WasmCloud.Native + + # Benchmarking common functions + # Helper function to run before benchmark tests + def pre_benchmark_run() do + # Set level to info to reduce log noise + Logger.configure(level: :info) + end + + # Helper function to run after benchmark tests + def post_benchmark_run() do + # Return log level to debug + Logger.configure(level: :debug) + end + + @spec run_benchmark( + test_config :: map(), + num_actors :: non_neg_integer(), + parallel :: list() | non_neg_integer(), + warmup :: non_neg_integer(), + time :: non_neg_integer() + ) :: :ok + # Run a benchmark with specified config, repeating for each parallel argument if it's a list + def run_benchmark(test_config, num_actors, parallel \\ [1], warmup \\ 1, time \\ 5) + when is_list(parallel) do + parallel + |> Enum.each(fn p -> run_benchmark(test_config, num_actors, p, warmup, time) end) + + :ok + end + + def run_benchmark(test_config, num_actors, parallel, warmup, time) + when is_number(parallel) do + IO.puts("Benchmarking with #{num_actors} actors and #{parallel} parallel requests") + pre_benchmark_run() + + Benchee.run(test_config, + warmup: warmup, + time: time, + parallel: parallel + ) + + post_benchmark_run() + :ok + end +end diff --git a/host_core/test/host_core/benchmark/echo_test.exs b/host_core/test/host_core/benchmark/echo_test.exs new file mode 100644 index 00000000..11b2685f --- /dev/null +++ b/host_core/test/host_core/benchmark/echo_test.exs @@ -0,0 +1,191 @@ +defmodule HostCore.Benchmark.EchoTest do + # We'd rather not run this test asynchronously because it's a benchmark. We'll get better + # results if this is the only test running at the time. + use ExUnit.Case, async: false + + alias HostCore.Actors.ActorRpcServer + alias HostCore.Actors.ActorSupervisor + alias HostCore.WasmCloud.Native + alias HostCore.Linkdefs.Manager + alias HostCore.Providers.ProviderSupervisor + + import HostCoreTest.Common, only: [cleanup: 2, standard_setup: 1, request_http: 2] + + @echo_key HostCoreTest.Constants.echo_key() + @echo_path HostCoreTest.Constants.echo_path() + @echo_wasi_key HostCoreTest.Constants.echo_wasi_key() + @echo_wasi_path HostCoreTest.Constants.echo_wasi_path() + + @httpserver_path HostCoreTest.Constants.httpserver_path() + @httpserver_key HostCoreTest.Constants.httpserver_key() + @httpserver_contract HostCoreTest.Constants.httpserver_contract() + @httpserver_link HostCoreTest.Constants.default_link() + + describe "Benchmarking actor invocations" do + setup :standard_setup + + test "load test with echo wasm32-unknown actor", %{ + :evt_watcher => evt_watcher, + :hconfig => config, + :host_pid => pid + } do + on_exit(fn -> cleanup(pid, config) end) + + # TODO: make configurable w/ benchee + num_actors = 1 + parallel = [1, 10] + + {msg, inv, port} = setup_echo_test(config, evt_watcher, @echo_key, @echo_path, num_actors) + + {:ok, _okay} = HTTPoison.start() + + test_config = %{ + "direct_echo_request" => fn -> + ActorRpcServer.request(msg) + end, + "nats_echo_request" => fn -> + config.lattice_prefix + |> HostCore.Nats.rpc_connection() + |> HostCore.Nats.safe_req(msg.topic, inv, receive_timeout: 2_000) + end, + "http_echo_request" => fn -> + {:ok, _resp} = request_http("http://localhost:#{port}/foo/bar", 1) + end + } + + HostCore.Benchmark.Common.run_benchmark(test_config, num_actors, parallel) + + assert true + end + + test "load test with echo wasm32-wasi actor", %{ + :evt_watcher => evt_watcher, + :hconfig => config, + :host_pid => pid + } do + on_exit(fn -> cleanup(pid, config) end) + + # TODO: make configurable w/ benchee + num_actors = 1 + parallel = [1, 10] + {:ok, bytes} = File.read(@echo_wasi_path) + + {:ok, _pids} = ActorSupervisor.start_actor(bytes, config.host_key, "", num_actors) + + seed = config.cluster_seed + + {msg, inv, port} = + setup_echo_test(config, evt_watcher, @echo_wasi_key, @echo_wasi_path, num_actors) + + {:ok, _okay} = HTTPoison.start() + + test_config = %{ + "direct_echo_request" => fn -> + ActorRpcServer.request(msg) + end, + "nats_echo_request" => fn -> + config.lattice_prefix + |> HostCore.Nats.rpc_connection() + |> HostCore.Nats.safe_req(msg.topic, inv, receive_timeout: 2_000) + end, + "http_echo_request" => fn -> + {:ok, _resp} = request_http("http://localhost:#{port}/foo/bar", 1) + end + } + + HostCore.Benchmark.Common.run_benchmark(test_config, num_actors, parallel) + + assert true + end + end + + @spec setup_echo_test( + config :: map(), + evt_watcher :: any(), + key :: binary(), + path :: binary(), + num_actors :: non_neg_integer() + ) :: + {msg :: map(), inv :: map(), port :: binary()} + # Helper function to set up echo tests and reduce code duplication + def setup_echo_test(config, evt_watcher, key, path, num_actors) do + {:ok, bytes} = File.read(path) + + {:ok, _pids} = ActorSupervisor.start_actor(bytes, config.host_key, "", num_actors) + + seed = config.cluster_seed + + req = + %{ + body: "hello", + header: %{}, + path: "/", + queryString: "", + method: "GET" + } + |> Msgpax.pack!() + |> IO.iodata_to_binary() + + port = "8081" + + # NOTE: Link definitions are put _before_ providers are started so that they receive + # the linkdef on startup. There is a race condition between provider starting and + # creating linkdef subscriptions that make this a desirable order for consistent tests. + + :ok = + Manager.put_link_definition( + config.lattice_prefix, + key, + @httpserver_contract, + @httpserver_link, + @httpserver_key, + %{PORT: port} + ) + + :ok = + HostCoreTest.EventWatcher.wait_for_linkdef( + evt_watcher, + key, + @httpserver_contract, + @httpserver_link + ) + + # Make sure we don't log too much out of the HTTPserver provider + System.put_env("RUST_LOG", "info,warp=warn") + + {:ok, _pid} = + ProviderSupervisor.start_provider_from_file( + config.host_key, + @httpserver_path, + @httpserver_link + ) + + :ok = + HostCoreTest.EventWatcher.wait_for_provider_start( + evt_watcher, + @httpserver_contract, + @httpserver_link, + @httpserver_key + ) + + inv = + Native.generate_invocation_bytes( + seed, + "system", + :provider, + @httpserver_key, + @httpserver_contract, + @httpserver_link, + "HttpServer.HandleRequest", + req + ) + + msg = %{ + body: IO.iodata_to_binary(inv), + topic: "wasmbus.rpc.#{config.lattice_prefix}.#{@echo_wasi_key}", + reply_to: "_INBOX.thisisatest.notinterested" + } + + {msg, inv, port} + end +end diff --git a/host_core/test/host_core/benchmark/kvcounter_test.exs b/host_core/test/host_core/benchmark/kvcounter_test.exs new file mode 100644 index 00000000..7d7ab205 --- /dev/null +++ b/host_core/test/host_core/benchmark/kvcounter_test.exs @@ -0,0 +1,181 @@ +defmodule HostCore.Benchmark.KvcounterTest do + # We'd rather not run this test asynchronously because it's a benchmark. We'll get better + # results if this is the only test running at the time. + use ExUnit.Case, async: false + + alias HostCore.Actors.ActorSupervisor + alias HostCore.WasmCloud.Native + alias HostCore.Linkdefs.Manager + alias HostCore.Providers.ProviderSupervisor + + import HostCoreTest.Common, only: [cleanup: 2, standard_setup: 1, request_http: 2] + + @kvcounter_key HostCoreTest.Constants.kvcounter_key() + @kvcounter_path HostCoreTest.Constants.kvcounter_path() + + @httpserver_contract HostCoreTest.Constants.httpserver_contract() + @httpserver_link HostCoreTest.Constants.default_link() + @httpserver_path HostCoreTest.Constants.httpserver_path() + @httpserver_key HostCoreTest.Constants.httpserver_key() + @keyvalue_contract HostCoreTest.Constants.keyvalue_contract() + @redis_link HostCoreTest.Constants.default_link() + @redis_path HostCoreTest.Constants.redis_path() + @redis_key HostCoreTest.Constants.redis_key() + + describe "Benchmarking actor invocations" do + setup :standard_setup + + test "load test with kvcounter wasm32-unknown actor", %{ + :evt_watcher => evt_watcher, + :hconfig => config, + :host_pid => pid + } do + on_exit(fn -> cleanup(pid, config) end) + + # TODO: make configurable w/ benchee + num_actors = 25 + parallel = [1, 10, 25, 50] + + {_msg, _inv, port} = + setup_kvcounter_test(config, evt_watcher, @kvcounter_key, @kvcounter_path, num_actors) + + {:ok, _okay} = HTTPoison.start() + + test_config = %{ + "http_kvcounter_request" => fn -> + {:ok, _resp} = request_http("http://localhost:#{port}/api/counter", 1) + end + } + + # Run the test at a few specified levels of parallelism, allowing for some warmup time to let compute calm down + HostCore.Benchmark.Common.run_benchmark(test_config, num_actors, parallel) + + assert true + end + end + + @spec setup_kvcounter_test( + config :: map(), + evt_watcher :: any(), + key :: binary(), + path :: binary(), + num_actors :: non_neg_integer() + ) :: + {msg :: map(), inv :: map(), port :: binary()} + # Helper function to set up kvcounter tests and reduce code duplication + def setup_kvcounter_test(config, evt_watcher, key, path, num_actors) do + {:ok, bytes} = File.read(path) + + {:ok, _pids} = ActorSupervisor.start_actor(bytes, config.host_key, "", num_actors) + + seed = config.cluster_seed + + req = + %{ + body: "hello", + header: %{}, + path: "/api/counter", + queryString: "", + method: "GET" + } + |> Msgpax.pack!() + |> IO.iodata_to_binary() + + port = "8082" + + # Make sure we don't log too much out of the HTTPserver provider + System.put_env("RUST_LOG", "info,warp=warn") + + :ok = HostCoreTest.EventWatcher.wait_for_actor_start(evt_watcher, key) + + # NOTE: Link definitions are put _before_ providers are started so that they receive + # the linkdef on startup. There is a race condition between provider starting and + # creating linkdef subscriptions that make this a desirable order for consistent tests. + + :ok = + Manager.put_link_definition( + config.lattice_prefix, + @kvcounter_key, + @httpserver_contract, + @httpserver_link, + @httpserver_key, + %{PORT: port} + ) + + :ok = + Manager.put_link_definition( + config.lattice_prefix, + @kvcounter_key, + @keyvalue_contract, + @redis_link, + @redis_key, + %{URL: "redis://127.0.0.1:6379"} + ) + + :ok = + HostCoreTest.EventWatcher.wait_for_linkdef( + evt_watcher, + @kvcounter_key, + @keyvalue_contract, + @redis_link + ) + + :ok = + HostCoreTest.EventWatcher.wait_for_linkdef( + evt_watcher, + @kvcounter_key, + @httpserver_contract, + @httpserver_link + ) + + {:ok, _pid} = + ProviderSupervisor.start_provider_from_file( + config.host_key, + @httpserver_path, + @httpserver_link + ) + + {:ok, _pid} = + ProviderSupervisor.start_provider_from_file( + config.host_key, + @redis_path, + @redis_link + ) + + :ok = + HostCoreTest.EventWatcher.wait_for_provider_start( + evt_watcher, + @keyvalue_contract, + @redis_link, + @redis_key + ) + + :ok = + HostCoreTest.EventWatcher.wait_for_provider_start( + evt_watcher, + @httpserver_contract, + @httpserver_link, + @httpserver_key + ) + + inv = + Native.generate_invocation_bytes( + seed, + "system", + :provider, + @httpserver_key, + @httpserver_contract, + @httpserver_link, + "HttpServer.HandleRequest", + req + ) + + msg = %{ + body: IO.iodata_to_binary(inv), + topic: "wasmbus.rpc.#{config.lattice_prefix}.#{@kvcounter_key}", + reply_to: "_INBOX.thisisatest.notinterested" + } + + {msg, inv, port} + end +end diff --git a/host_core/test/host_core/benchmark_test.exs b/host_core/test/host_core/benchmark_test.exs deleted file mode 100644 index 0bc41047..00000000 --- a/host_core/test/host_core/benchmark_test.exs +++ /dev/null @@ -1,92 +0,0 @@ -defmodule HostCore.BenchmarkTest do - # We'd rather not run this test asynchronously because it's a benchmark. We'll get better - # results if this is the only test running at the time. - use ExUnit.Case, async: false - - alias HostCore.Actors.ActorRpcServer - alias HostCore.Actors.ActorSupervisor - alias HostCore.WasmCloud.Native - - import HostCoreTest.Common, only: [cleanup: 2, standard_setup: 1] - - @echo_key HostCoreTest.Constants.echo_key() - @echo_path HostCoreTest.Constants.echo_path() - - @httpserver_key HostCoreTest.Constants.httpserver_key() - @httpserver_contract HostCoreTest.Constants.httpserver_contract() - @httpserver_link HostCoreTest.Constants.default_link() - - describe "Benchmarking actor invocations" do - setup :standard_setup - - test "load test with echo actor", %{ - :evt_watcher => _evt_watcher, - :hconfig => config, - :host_pid => pid - } do - on_exit(fn -> cleanup(pid, config) end) - - num_actors = 10 - parallel = 1 - {:ok, bytes} = File.read(@echo_path) - - {:ok, _pids} = ActorSupervisor.start_actor(bytes, config.host_key, "", num_actors) - - seed = config.cluster_seed - - req = - %{ - body: "hello", - header: %{}, - path: "/", - queryString: "", - method: "GET" - } - |> Msgpax.pack!() - |> IO.iodata_to_binary() - - inv = - Native.generate_invocation_bytes( - seed, - "system", - :provider, - @httpserver_key, - @httpserver_contract, - @httpserver_link, - "HttpServer.HandleRequest", - req - ) - - msg = %{ - body: IO.iodata_to_binary(inv), - topic: "wasmbus.rpc.#{config.lattice_prefix}.#{@echo_key}", - reply_to: "_INBOX.thisisatest.notinterested" - } - - IO.puts("Benchmarking with #{num_actors} actors and #{parallel} parallel requests") - # very noisy debug logs during bench - Logger.configure(level: :info) - - Benchee.run( - %{ - "nats_echo_request" => fn -> - config.lattice_prefix - |> HostCore.Nats.rpc_connection() - |> HostCore.Nats.safe_req(msg.topic, inv, receive_timeout: 2_000) - end, - "direct_echo_request" => fn -> - ActorRpcServer.request(msg) - end - }, - warmup: 1, - time: 5, - parallel: parallel - ) - - # turning debug logs back on - Logger.configure(level: :debug) - - assert true - end - end -end diff --git a/host_core/test/test_helper.exs b/host_core/test/test_helper.exs index 764f7b22..91a324f2 100644 --- a/host_core/test/test_helper.exs +++ b/host_core/test/test_helper.exs @@ -1,3 +1,4 @@ +Code.require_file("host_core/benchmark/common.exs", __DIR__) Code.require_file("support/event_watcher.exs", __DIR__) Code.require_file("support/common.exs", __DIR__) Code.require_file("support/constants.exs", __DIR__)