Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial commit for distributed tests using crank (#7323)
- Loading branch information
1 parent
43cce69
commit 1ae2434
Showing
32 changed files
with
1,515 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,6 +30,7 @@ x86/ | |
| bld/ | ||
| [Bb]in/ | ||
| [Oo]bj/ | ||
| Artifacts/ | ||
|
|
||
| # Roslyn cache directories | ||
| *.ide/ | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| variables: | ||
| clusterId: '{{ "now" | date: "%s" }}' | ||
| serviceId: '{{ "now" | date: "%s" }}' | ||
| secretSource: KeyVault | ||
| framework: net5.0 | ||
|
|
||
| jobs: | ||
| server: | ||
| source: | ||
| localFolder: Artifacts/DistributedTests/DistributedTests.Server/{{framework}} | ||
| executable: DistributedTests.Server.exe | ||
| readyStateText: Orleans Silo started. | ||
| framework: net5.0 | ||
| arguments: "{{configurator}} --clusterId {{clusterId}} --serviceId {{serviceId}} --secretSource {{secretSource}} {{configuratorOptions}}" | ||
| onConfigure: | ||
| - if (job.endpoints.Count > 0) { | ||
| job.endpoints.Reverse(); | ||
| job.endpoints.RemoveRange(job.variables.instances, job.endpoints.Count - job.variables.instances); | ||
| } | ||
| client: | ||
| source: | ||
| localFolder: Artifacts/DistributedTests/DistributedTests.Client/{{framework}} | ||
| executable: DistributedTests.Client.exe | ||
| waitForExit: true | ||
| framework: net5.0 | ||
| arguments: "{{command}} --clusterId {{clusterId}} --serviceId {{serviceId}} --secretSource {{secretSource}} {{commandOptions}}" | ||
| onConfigure: | ||
| - if (job.endpoints.Count > 0) { | ||
| job.endpoints.Reverse(); | ||
| job.endpoints.RemoveRange(job.variables.instances, job.endpoints.Count - job.variables.instances); | ||
| } | ||
|
|
||
| scenarios: | ||
| ping: | ||
| server: | ||
| job: server | ||
| variables: | ||
| instances: 10 | ||
| configurator: SimpleSilo | ||
| client: | ||
| job: client | ||
| variables: | ||
| command: ping | ||
| instances: 1 | ||
| numWorkers: 250 | ||
| blocksPerWorker: 0 | ||
| requestsPerBlock: 500 | ||
| duration: 120 | ||
| commandOptions: "--numWorkers {{numWorkers}} --blocksPerWorker {{blocksPerWorker}} --requestsPerBlock {{requestsPerBlock}} --duration {{duration}}" | ||
| streaming: | ||
| server: | ||
| job: server | ||
| variables: | ||
| instances: 10 | ||
| configurator: EventGeneratorStreamingSilo | ||
| duration: 300 | ||
| type: ExplicitGrainBasedAndImplicit | ||
| streamsPerQueue: 1000 | ||
| queueCount: 8 | ||
| wait: 20 | ||
| configuratorOptions: "--type {{type}} --streamsPerQueue {{streamsPerQueue}} --queueCount {{queueCount}} --wait {{wait}} --duration {{duration}}" | ||
| client: | ||
| job: client | ||
| variables: | ||
| command: counter | ||
| instances: 1 | ||
| commandOptions: "requests errors" | ||
| reliability: | ||
| server: | ||
| job: server | ||
| variables: | ||
| instances: 10 | ||
| configurator: SimpleSilo | ||
| chaosagent: | ||
| job: client | ||
| waitForExit: false | ||
| variables: | ||
| command: chaosagent | ||
| instances: 1 | ||
| wait: 30 | ||
| serversPerRound: 1 | ||
| rounds: 4 | ||
| roundDelay: 30 | ||
| graceful: true | ||
| restart: true | ||
| commandOptions: "--wait {{wait}} --serversPerRound {{serversPerRound}} --rounds {{rounds}} --roundDelay {{roundDelay}} --graceful {{graceful}} --restart {{restart}}" | ||
| client: | ||
| job: client | ||
| waitForExit: true | ||
| variables: | ||
| command: ping | ||
| instances: 1 | ||
| numWorkers: 250 | ||
| blocksPerWorker: 0 | ||
| requestsPerBlock: 500 | ||
| duration: 180 | ||
| commandOptions: "--numWorkers {{numWorkers}} --blocksPerWorker {{blocksPerWorker}} --requestsPerBlock {{requestsPerBlock}} --duration {{duration}}" | ||
|
|
||
| profiles: | ||
| local: | ||
| variables: | ||
| secretSource: File | ||
| jobs: | ||
| server: | ||
| endpoints: | ||
| - http://localhost:5010 | ||
| variables: | ||
| instances: 1 | ||
| client: | ||
| endpoints: | ||
| - http://localhost:5010 | ||
| chaosagent: | ||
| endpoints: | ||
| - http://localhost:5010 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 77 additions & 0 deletions
77
test/DistributedTests/DistributedTests.Client/Commands/ChaosAgentCommand.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.CommandLine; | ||
| using System.CommandLine.Invocation; | ||
| using System.Linq; | ||
| using System.Threading; | ||
| using System.Threading.Tasks; | ||
| using DistributedTests.Common.MessageChannel; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| namespace DistributedTests.Client.Commands | ||
| { | ||
| public class ChaosAgentCommand : Command | ||
| { | ||
| private readonly ILogger _logger; | ||
|
|
||
| private class Parameters | ||
| { | ||
| public string ServiceId { get; set; } | ||
| public string ClusterId { get; set; } | ||
| public SecretConfiguration.SecretSource SecretSource { get; set; } | ||
| public int Wait { get; set; } | ||
| public int ServersPerRound { get; set; } | ||
| public int Rounds { get; set; } | ||
| public int RoundDelay { get; set; } | ||
| public bool Graceful { get; set; } | ||
| public bool Restart { get; set; } | ||
| } | ||
|
|
||
| public ChaosAgentCommand(ILogger logger) | ||
| : base("chaosagent", "Shutdown/restart servers gracefully or not") | ||
| { | ||
| AddOption(OptionHelper.CreateOption<string>("--serviceId", isRequired: true)); | ||
| AddOption(OptionHelper.CreateOption<string>("--clusterId", isRequired: true)); | ||
| AddOption(OptionHelper.CreateOption("--secretSource", defaultValue: SecretConfiguration.SecretSource.File)); | ||
| AddOption(OptionHelper.CreateOption<int>("--wait", defaultValue: 30)); | ||
| AddOption(OptionHelper.CreateOption<int>("--serversPerRound", defaultValue: 1)); | ||
| AddOption(OptionHelper.CreateOption<int>("--rounds", defaultValue: 5)); | ||
| AddOption(OptionHelper.CreateOption<int>("--roundDelay", defaultValue: 60)); | ||
| AddOption(OptionHelper.CreateOption<bool>("--graceful", defaultValue: false)); | ||
| AddOption(OptionHelper.CreateOption<bool>("--restart", defaultValue: false)); | ||
|
|
||
| Handler = CommandHandler.Create<Parameters>(RunAsync); | ||
| _logger = logger; | ||
| } | ||
|
|
||
| private async Task RunAsync(Parameters parameters) | ||
| { | ||
| var secrets = SecretConfiguration.Load(parameters.SecretSource); | ||
| var channel = await Channels.CreateSendChannel(parameters.ClusterId, secrets); | ||
|
|
||
| _logger.LogInformation($"Waiting {parameters.Wait} seconds before starting..."); | ||
| await Task.Delay(TimeSpan.FromSeconds(parameters.Wait)); | ||
|
|
||
| for (var i=0; i<parameters.Rounds; i++) | ||
| { | ||
| _logger.LogInformation($"Round #{i + 1}: sending {parameters.ServersPerRound} orders [Restart: {parameters.Restart}, Graceful: {parameters.Graceful}]"); | ||
| var responses = await channel.SendMessages( | ||
| GetMessages(), | ||
| new CancellationTokenSource(TimeSpan.FromSeconds(parameters.RoundDelay)).Token); | ||
| _logger.LogInformation($"Round #{i + 1}: silos {string.Join(",", responses.Select(r => r.ServerName))} acked"); | ||
| _logger.LogInformation($"Round #{i + 1}: waiting {parameters.RoundDelay}"); | ||
| await Task.Delay(TimeSpan.FromSeconds(parameters.RoundDelay)); | ||
| } | ||
|
|
||
| List<ServerMessage> GetMessages() | ||
| { | ||
| var msgs = new List<ServerMessage>(); | ||
| for (var i = 0; i < parameters.ServersPerRound; i++) | ||
| { | ||
| msgs.Add(new ServerMessage(parameters.Graceful, parameters.Restart)); | ||
| } | ||
| return msgs; | ||
| } | ||
| } | ||
| } | ||
| } |
87 changes: 87 additions & 0 deletions
87
test/DistributedTests/DistributedTests.Client/Commands/CounterCaptureCommand.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.CommandLine; | ||
| using System.CommandLine.Invocation; | ||
| using System.Threading.Tasks; | ||
| using DistributedTests.GrainInterfaces; | ||
| using Microsoft.Crank.EventSources; | ||
| using Microsoft.Extensions.DependencyInjection; | ||
| using Microsoft.Extensions.Hosting; | ||
| using Microsoft.Extensions.Logging; | ||
| using Orleans; | ||
| using Orleans.Configuration; | ||
| using Orleans.Hosting; | ||
|
|
||
| namespace DistributedTests.Client.Commands | ||
| { | ||
| public class CounterCaptureCommand : Command | ||
| { | ||
| private readonly ILogger _logger; | ||
|
|
||
| private class Parameters | ||
| { | ||
| public string ServiceId { get; set; } | ||
| public string ClusterId { get; set; } | ||
| public SecretConfiguration.SecretSource SecretSource { get; set; } | ||
| public string CounterKey { get; set; } | ||
| public List<string> Counters { get; set; } | ||
| } | ||
|
|
||
| public CounterCaptureCommand(ILogger logger) | ||
| : base("counter", "capture the counters in parameter") | ||
| { | ||
| AddOption(OptionHelper.CreateOption<string>("--serviceId", isRequired: true)); | ||
| AddOption(OptionHelper.CreateOption<string>("--clusterId", isRequired: true)); | ||
| AddOption(OptionHelper.CreateOption("--secretSource", defaultValue: SecretConfiguration.SecretSource.File)); | ||
| AddOption(OptionHelper.CreateOption<string>("--counterKey", defaultValue: StreamingConstants.DefaultCounterGrain)); | ||
| AddArgument(new Argument<List<string>>("Counters") { Arity = ArgumentArity.OneOrMore }); | ||
|
|
||
| Handler = CommandHandler.Create<Parameters>(RunAsync); | ||
| _logger = logger; | ||
| } | ||
|
|
||
| private async Task RunAsync(Parameters parameters) | ||
| { | ||
| _logger.LogInformation("Connecting to cluster..."); | ||
| var secrets = SecretConfiguration.Load(parameters.SecretSource); | ||
| var hostBuilder = new HostBuilder() | ||
| .UseOrleansClient(builder => { | ||
| builder | ||
| .Configure<ClusterOptions>(options => { options.ClusterId = parameters.ClusterId; options.ServiceId = parameters.ServiceId; }) | ||
| .UseAzureStorageClustering(options => options.ConfigureTableServiceClient(secrets.ClusteringConnectionString)); | ||
| }); | ||
| using var host = hostBuilder.Build(); | ||
| await host.StartAsync(); | ||
|
|
||
| var client = host.Services.GetService<IClusterClient>(); | ||
|
|
||
| var counterGrain = client.GetGrain<ICounterGrain>(parameters.CounterKey); | ||
|
|
||
| var duration = await counterGrain.GetRunDuration(); | ||
| BenchmarksEventSource.Register("duration", Operations.First, Operations.Last, "duration", "duration", "n0"); | ||
| BenchmarksEventSource.Measure("duration", duration.TotalSeconds); | ||
|
|
||
| var initialWait = await counterGrain.WaitTimeForReport(); | ||
|
|
||
| _logger.LogInformation($"Counters should be ready in {initialWait}"); | ||
| await Task.Delay(initialWait); | ||
|
|
||
| _logger.LogInformation($"Counters ready"); | ||
| foreach (var counter in parameters.Counters) | ||
| { | ||
| var value = await counterGrain.GetTotalCounterValue(counter); | ||
| _logger.LogInformation($"{counter}: {value}"); | ||
| BenchmarksEventSource.Register(counter, Operations.First, Operations.Sum, counter, counter, "n0"); | ||
| BenchmarksEventSource.Measure(counter, value); | ||
| if (string.Equals(counter, "requests", StringComparison.InvariantCultureIgnoreCase)) | ||
| { | ||
| var rps = (float) value / duration.TotalSeconds; | ||
| BenchmarksEventSource.Register("rps", Operations.First, Operations.Last, "rps", "Requests per second", "n0"); | ||
| BenchmarksEventSource.Measure("rps", rps); | ||
| } | ||
| } | ||
|
|
||
| await host.StopAsync(); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.