Simple benchmarker (#69)

Fixes #54

.github/workflows/nightly.yml

@@ -0,0 +1,10 @@
+name: Nightly
+
+on:
+  schedule:
+    # (12 AM PST)
+    - cron: "00 07 * * *"
+
+jobs:
+  nightly:
+    uses: ./.github/workflows/run-bench.yml

.github/workflows/run-bench.yml

@@ -0,0 +1,56 @@
+name: Run Bench
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  run-bench:
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+
+      - name: Setup Rust cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: src/Temporalio/Bridge
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+
+      - name: Install protoc
+        uses: arduino/setup-protoc@v1
+        with:
+          # TODO(cretz): Upgrade when https://github.com/arduino/setup-protoc/issues/33 fixed
+          version: '3.x'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build
+        run: dotnet build --configuration Release
+
+      # Run a bunch of bench tests. We run multiple times since results vary.
+
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 100 --max-cached-workflows 100 --max-concurrent 100
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 100 --max-cached-workflows 100 --max-concurrent 100
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 100 --max-cached-workflows 100 --max-concurrent 100
+
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 1000 --max-cached-workflows 1000 --max-concurrent 1000
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 1000 --max-cached-workflows 1000 --max-concurrent 1000
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 1000 --max-cached-workflows 1000 --max-concurrent 1000
+
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 1000 --max-cached-workflows 100 --max-concurrent 100
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 1000 --max-cached-workflows 100 --max-concurrent 100
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 1000 --max-cached-workflows 100 --max-concurrent 100
+
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 10000 --max-cached-workflows 10000 --max-concurrent 10000
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 10000 --max-cached-workflows 10000 --max-concurrent 10000
+
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 10000 --max-cached-workflows 1000 --max-concurrent 1000
+      - run: dotnet run --configuration Release --project tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj -- --workflow-count 10000 --max-cached-workflows 1000 --max-concurrent 1000

tests/Temporalio.SimpleBench/.editorconfig

@@ -0,0 +1,28 @@
+[*.cs]
+
+##### Temporal additions ######
+
+# Please keep in alphabetical order by field.
+
+# Don't need static workflow run call
+dotnet_diagnostic.CA1822.severity = none
+
+# Don't need to mark bench classes sealed
+dotnet_diagnostic.CA1852.severity = none
+
+# Don't need ConfigureAwait
+dotnet_diagnostic.CA2007.severity = none
+
+# Don't need docs
+dotnet_diagnostic.CS1591.severity = none
+
+# Can have multiple types in one file
+dotnet_diagnostic.SA1402.severity = none
+
+# Don't need docs
+dotnet_diagnostic.SA1600.severity = none
+
+# Program.cs doesn't need to match name
+dotnet_diagnostic.SA1649.severity = none
+
+###############################

tests/Temporalio.SimpleBench/Program.cs

@@ -0,0 +1,155 @@
+using System.CommandLine;
+using System.Diagnostics;
+using Microsoft.Extensions.Logging;
+using Temporalio.Activities;
+using Temporalio.SimpleBench;
+using Temporalio.Testing;
+using Temporalio.Worker;
+using Temporalio.Workflows;
+
+// Build command
+var cmd = new RootCommand("Simple bench runner");
+var workflowCountOption = new Option<int>("--workflow-count", "Number of workflows")
+{
+    IsRequired = true,
+};
+cmd.AddOption(workflowCountOption);
+var maxCachedWorkflowsOption = new Option<int>("--max-cached-workflows", "Number of workflows cached")
+{
+    IsRequired = true,
+};
+cmd.AddOption(maxCachedWorkflowsOption);
+var maxConcurrentOption = new Option<int>("--max-concurrent", "Number of concurrent workflows/activities")
+{
+    IsRequired = true,
+};
+cmd.AddOption(maxConcurrentOption);
+using var loggerFactory = LoggerFactory.Create(builder =>
+    builder.AddSimpleConsole().SetMinimumLevel(LogLevel.Information));
+var logger = loggerFactory.CreateLogger<Program>();
+
+// Set handler
+cmd.SetHandler(async ctx =>
+{
+    var workflowCount = ctx.ParseResult.GetValueForOption(workflowCountOption);
+    var maxCachedWorkflows = ctx.ParseResult.GetValueForOption(maxCachedWorkflowsOption);
+    var maxConcurrent = ctx.ParseResult.GetValueForOption(maxConcurrentOption);
+
+    // Start server
+    logger.LogInformation("Starting local environment");
+    await using var env = await WorkflowEnvironment.StartLocalAsync(
+        new() { LoggerFactory = loggerFactory });
+    var taskQueue = $"task-queue-{Guid.NewGuid()}";
+
+    // Create a bunch of workflows
+    logger.LogInformation("Starting {WorkflowCount} workflows", workflowCount);
+    var startWatch = new Stopwatch();
+    startWatch.Start();
+    var handles = await Task.WhenAll(Enumerable.Range(0, workflowCount).Select(index =>
+        env.Client.StartWorkflowAsync(
+            BenchWorkflow.Ref.RunAsync,
+            $"user-{index}",
+            new($"workflow-{index}-{Guid.NewGuid()}", taskQueue))));
+    startWatch.Stop();
+
+    // Start a worker to run them
+    logger.LogInformation("Starting worker");
+    using var worker = new TemporalWorker(
+        env.Client,
+        new TemporalWorkerOptions(taskQueue)
+        {
+            MaxCachedWorkflows = maxCachedWorkflows,
+            MaxConcurrentWorkflowTasks = maxConcurrent,
+            MaxConcurrentActivities = maxConcurrent,
+        }.
+            AddActivity(BenchActivities.BenchActivity).
+            AddWorkflow(typeof(BenchWorkflow)));
+    using var cancelSource = CancellationTokenSource.CreateLinkedTokenSource(ctx.GetCancellationToken());
+    var workerTask = Task.Run(() => worker.ExecuteAsync(cancelSource.Token));
+
+    // Wait for all workflows
+    var resultWatch = new Stopwatch();
+    var memoryTask = Task.Run(() => MemoryTracker.TrackMaxMemoryBytesAsync(cancelSource.Token));
+    resultWatch.Start();
+    foreach (var handle in handles)
+    {
+        await handle.GetResultAsync();
+    }
+    resultWatch.Stop();
+
+    // Cancel worker and wait for cancelled
+    cancelSource.Cancel();
+    try
+    {
+        await workerTask;
+    }
+    catch (OperationCanceledException)
+    {
+    }
+    var maxMem = await memoryTask;
+
+    // Dump results
+    logger.LogInformation("Results: {Results}", new Results(
+        WorkflowCount: workflowCount,
+        MaxCachedWorkflows: maxCachedWorkflows,
+        MaxConcurrent: maxConcurrent,
+        MaxMemoryMib: (long)Math.Round(maxMem / Math.Pow(1024, 2)),
+        StartDuration: startWatch.Elapsed,
+        ResultDuration: resultWatch.Elapsed,
+        WorkflowsPerSecond: Math.Round(workflowCount / (decimal)resultWatch.Elapsed.TotalSeconds, 2)));
+});
+
+// Run command
+await cmd.InvokeAsync(args);
+
+namespace Temporalio.SimpleBench
+{
+    public static class MemoryTracker
+    {
+        public static async Task<long> TrackMaxMemoryBytesAsync(CancellationToken cancel)
+        {
+            // Get the memory every 800ms
+            var process = Process.GetCurrentProcess();
+            var max = -1L;
+            while (!cancel.IsCancellationRequested)
+            {
+                // We don't want to cancel delay ever, always let it finish
+                await Task.Delay(800, CancellationToken.None);
+                var curr = process.WorkingSet64;
+                if (curr > max)
+                {
+                    max = curr;
+                }
+            }
+            return max;
+        }
+    }
+
+    public static class BenchActivities
+    {
+        [Activity]
+        public static string BenchActivity(string name) => $"Hello, {name}";
+    }
+
+    [Workflow]
+    public class BenchWorkflow
+    {
+        public static readonly BenchWorkflow Ref = WorkflowRefs.Create<BenchWorkflow>();
+
+        [WorkflowRun]
+        public async Task<string> RunAsync(string name)
+        {
+            return await Workflow.ExecuteActivityAsync(
+                BenchActivities.BenchActivity, name, new() { StartToCloseTimeout = TimeSpan.FromSeconds(30) });
+        }
+    }
+
+    public record Results(
+        int WorkflowCount,
+        int MaxCachedWorkflows,
+        int MaxConcurrent,
+        long MaxMemoryMib,
+        TimeSpan StartDuration,
+        TimeSpan ResultDuration,
+        decimal WorkflowsPerSecond);
+}

tests/Temporalio.SimpleBench/Temporalio.SimpleBench.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Temporalio\Temporalio.csproj" />
+    <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
+    <PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="7.0.0" />
+  </ItemGroup>
+
+</Project>