Activation Rebalancing

Orleans.sln

@@ -242,6 +242,12 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Orleans.Serialization.FShar
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Serialization.MessagePack", "src\Orleans.Serialization.MessagePack\Orleans.Serialization.MessagePack.csproj", "{F50F81B6-E9B5-4143-B66B-A1AD913F6E9C}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ActivationRebalancing", "ActivationRebalancing", "{B0DC8B8D-29CD-4CA3-A874-471F75595829}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ActivationRebalancing.Cluster", "playground\ActivationRebalancing\ActivationRebalancing.Cluster\ActivationRebalancing.Cluster.csproj", "{2D109E60-E9BF-4F57-BBCD-DF5FA7768B00}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ActivationRebalancing.Frontend", "playground\ActivationRebalancing\ActivationRebalancing.Frontend\ActivationRebalancing.Frontend.csproj", "{DFAF9FFC-EBD9-45F0-A121-010D29A296C1}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -632,6 +638,14 @@ Global
 		{F50F81B6-E9B5-4143-B66B-A1AD913F6E9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F50F81B6-E9B5-4143-B66B-A1AD913F6E9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F50F81B6-E9B5-4143-B66B-A1AD913F6E9C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2D109E60-E9BF-4F57-BBCD-DF5FA7768B00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2D109E60-E9BF-4F57-BBCD-DF5FA7768B00}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2D109E60-E9BF-4F57-BBCD-DF5FA7768B00}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2D109E60-E9BF-4F57-BBCD-DF5FA7768B00}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DFAF9FFC-EBD9-45F0-A121-010D29A296C1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DFAF9FFC-EBD9-45F0-A121-010D29A296C1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DFAF9FFC-EBD9-45F0-A121-010D29A296C1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DFAF9FFC-EBD9-45F0-A121-010D29A296C1}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -748,6 +762,9 @@ Global
 		{84B44F1D-B7FE-40E3-82F0-730A55AC8613} = {316CDCC7-323F-4264-9FC9-667662BB1F80}
 		{B2D53D3C-E44A-4C9B-AAEE-28FB8C1BDF62} = {A6573187-FD0D-4DF7-91D1-03E07E470C0A}
 		{F50F81B6-E9B5-4143-B66B-A1AD913F6E9C} = {4CD3AA9E-D937-48CA-BB6C-158E12257D23}
+		{B0DC8B8D-29CD-4CA3-A874-471F75595829} = {A41DE3D1-F8AA-4234-BE6F-3C9646A1507A}
+		{2D109E60-E9BF-4F57-BBCD-DF5FA7768B00} = {B0DC8B8D-29CD-4CA3-A874-471F75595829}
+		{DFAF9FFC-EBD9-45F0-A121-010D29A296C1} = {B0DC8B8D-29CD-4CA3-A874-471F75595829}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {7BFB3429-B5BB-4DB1-95B4-67D77A864952}

playground/ActivationRebalancing/ActivationRebalancing.Cluster/ActivationRebalancing.Cluster.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <OrleansBuildTimeCodeGen>true</OrleansBuildTimeCodeGen>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Orleans.Server\Orleans.Server.csproj" />
+  </ItemGroup>
+
+</Project>

playground/ActivationRebalancing/ActivationRebalancing.Cluster/Program.cs

@@ -0,0 +1,170 @@
+using System.Diagnostics;
+using System.Net;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+using Orleans.Configuration;
+using Orleans.Runtime.Placement;
+
+#nullable enable
+
+// Ledjon: The silos will run in the same process so they will have the same memory usage.
+// I previously had 4 console apps to run the example, but didn't want to add so many proj into the solution.
+// I am sure with something like Aspire that would be easier, but for now I'll leave them like this.
+// You (the reader) feel free to run this in different processes for a more realistic example.
+
+var host0 = await StartSiloHost(0);
+var host1 = await StartSiloHost(1);
+var host2 = await StartSiloHost(2);
+var host3 = await StartSiloHost(3);
+IHost? host5 = null;
+
+Console.WriteLine("All silos have started.");
+
+var grainFactory = host0.Services.GetRequiredService<IGrainFactory>();
+var mgmtGrain = grainFactory.GetGrain<IManagementGrain>(0);
+
+var silos = await mgmtGrain.GetHosts(onlyActive: true);
+Debug.Assert(silos.Count == 4);
+var addresses = silos.Select(x => x.Key).ToArray();
+
+var tasks = new List<Task>();
+RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[0]);
+for (var i = 0; i < 300; i++)
+{
+    tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+}
+
+RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[1]);
+for (var i = 0; i < 30; i++)
+{
+    tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+}
+
+RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[2]);
+for (var i = 0; i < 410; i++)
+{
+    tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+}
+
+RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[3]);
+for (var i = 0; i < 120; i++)
+{
+    tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+}
+
+var sessionCount = 0;
+while (true)
+{
+    if (sessionCount == 25)
+    {
+        RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[0]);
+        for (var i = 0; i < 50; i++)
+        {
+            tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+        }
+
+        RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[1]);
+        for (var i = 0; i < 50; i++)
+        {
+            tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+        }
+    }
+
+    if (sessionCount == 35)
+    {
+        RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[1]);
+        for (var i = 0; i < 50; i++)
+        {
+            tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+        }
+
+        RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[2]);
+        for (var i = 0; i < 50; i++)
+        {
+            tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+        }
+    }
+
+    if (sessionCount == 40)
+    {
+        host5 = await StartSiloHost(4);
+    }
+
+    if (sessionCount == 45)
+    {
+        RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[2]);
+        for (var i = 0; i < 50; i++)
+        {
+            tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+        }
+
+        RequestContext.Set(IPlacementDirector.PlacementHintKey, addresses[3]);
+        for (var i = 0; i < 50; i++)
+        {
+            tasks.Add(grainFactory.GetGrain<IRebalancingTestGrain>(Guid.NewGuid()).Ping());
+        }
+    }
+
+    await Task.Delay(5000); // session duration
+    sessionCount++;
+
+    if (sessionCount > 55)
+    {
+        break;
+    }
+}
+
+Console.WriteLine("Simulation has finished. Press Enter to terminate...");
+Console.ReadLine();
+
+await host0.StopAsync();
+await host1.StopAsync();
+await host2.StopAsync();
+await host3.StopAsync();
+
+if (host5 != null)
+{
+    await host5.StopAsync();
+}
+
+static async Task<IHost> StartSiloHost(int num)
+{
+    #pragma warning disable ORLEANSEXP002
+        var host = Host.CreateDefaultBuilder()
+            .ConfigureLogging(builder => builder
+                .AddFilter("", LogLevel.Error)
+                .AddFilter("Orleans.Runtime.Placement.Rebalancing", LogLevel.Trace)
+                .AddConsole())
+            .UseOrleans(builder => builder
+                .Configure<ActivationRebalancerOptions>(o =>
+                {
+                    o.RebalancerDueTime = TimeSpan.FromSeconds(5);
+                    o.SessionCyclePeriod = TimeSpan.FromSeconds(5);
+                    // uncomment these below, if you want higher migration rate
+                    //o.CycleNumberWeight = 1;
+                    //o.SiloNumberWeight = 0; 
+                })
+                .UseLocalhostClustering(
+                    siloPort: EndpointOptions.DEFAULT_SILO_PORT + num,
+                    gatewayPort: EndpointOptions.DEFAULT_GATEWAY_PORT + num,
+                    primarySiloEndpoint: new IPEndPoint(IPAddress.Loopback, EndpointOptions.DEFAULT_SILO_PORT))
+                .AddActivationRebalancer())
+            .Build();
+    #pragma warning restore ORLEANSEXP002
+
+    await host.StartAsync();
+    Console.WriteLine($"Silo{num} started.");
+
+    return host;
+}
+
+public interface IRebalancingTestGrain : IGrainWithGuidKey
+{
+    Task Ping();
+}
+
+public class RebalancingTestGrain : Grain, IRebalancingTestGrain
+{
+    public Task Ping() => Task.CompletedTask;
+}

playground/ActivationRebalancing/ActivationRebalancing.Frontend/ActivationRebalancing.Frontend.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Orleans.Server\Orleans.Server.csproj" />
+    <ProjectReference Include="..\ActivationRebalancing.Cluster\ActivationRebalancing.Cluster.csproj" />
+  </ItemGroup>
+
+</Project>

playground/ActivationRebalancing/ActivationRebalancing.Frontend/Controllers/StatsController.cs

@@ -0,0 +1,36 @@
+using Microsoft.AspNetCore.Mvc;
+using Orleans.Runtime;
+using Orleans;
+
+namespace ActivationRebalancing.Frontend.Controllers;
+
+[ApiController]
+[Route("api/[controller]")]
+public class StatsController(IClusterClient clusterClient) : ControllerBase
+{
+    [HttpGet("silos")]
+    public async Task<IActionResult> GetStats()
+    {
+        var grainStats = await clusterClient
+            .GetGrain<IManagementGrain>(0)
+            .GetDetailedGrainStatistics();
+
+        var siloData = grainStats.GroupBy(stat => stat.SiloAddress)
+            .Select(g => new SiloData(g.Key.ToString(), g.Count()))
+            .ToList();
+
+        if (siloData.Count == 4)
+        {
+            siloData = [.. siloData, new SiloData("x", 0)];
+        }
+
+        if (siloData.Count > 5)
+        {
+            throw new NotSupportedException("The frontend cant support more than 6 silos");
+        }
+
+        return Ok(siloData);
+    }
+}
+
+public record SiloData(string Host, int Activations);

playground/ActivationRebalancing/ActivationRebalancing.Frontend/Program.cs

@@ -0,0 +1,17 @@
+using Orleans.Hosting;
+
+var builder = WebApplication.CreateBuilder(args);
+
+builder.UseOrleansClient(clientBuilder => clientBuilder.UseLocalhostClustering());
+builder.Services.AddControllers();
+
+var app = builder.Build();
+
+var options = new DefaultFilesOptions();
+options.DefaultFileNames.Clear();
+options.DefaultFileNames.Add("index.html");
+
+app.UseDefaultFiles(options);
+app.UseStaticFiles();
+app.MapControllers();
+app.Run();

playground/ActivationRebalancing/ActivationRebalancing.Frontend/Properties/launchSettings.json

@@ -0,0 +1,14 @@
+{
+  "profiles": {
+    "http": {
+      "commandName": "Project",
+      "dotnetRunMessages": true,
+      "launchBrowser": true,
+      "launchUrl": "index.html",
+      "applicationUrl": "http://localhost:5000",
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

playground/ActivationRebalancing/ActivationRebalancing.Frontend/appsettings.Development.json

@@ -0,0 +1,8 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  }
+}

playground/ActivationRebalancing/ActivationRebalancing.Frontend/appsettings.json

@@ -0,0 +1,9 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft.AspNetCore": "Warning"
+    }
+  },
+  "AllowedHosts": "*"
+}