Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<PackageVersion Include="Microsoft.Build.Utilities.Core" Version="17.14.28" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.14.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.1" />
<PackageVersion Include="Microsoft.Testing.Platform" Version="2.0.2" />
<PackageVersion Include="Mono.Cecil" Version="0.11.6" />
<PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.1" />
</ItemGroup>
Expand Down
26 changes: 25 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,9 @@ dotnet run --project tests\Clockwork.Tests\Clockwork.Tests.csproj -- --timeout 6
dotnet pack src/Clockwork/Clockwork.csproj --configuration Release
```

The NuGet package ID is `Clockwork.Simulation`. Until packages are published, clone the repository or add it as a Git submodule and reference `src/Clockwork/Clockwork.csproj`.
The core NuGet package ID is `Clockwork.Simulation`; test-runner integration and replay helpers are
in `Clockwork.Simulation.Testing`. Until packages are published, clone the repository or add it as a
Git submodule and reference the corresponding project under `src`.

## Instrumented simulation test projects

Expand Down Expand Up @@ -244,6 +246,28 @@ including the stop reason, counters, limits, and pending-work snapshot. They sha
engine, so time advancement and stuck detection remain consistent. Every drive method requires a
`CancellationToken` and observes it between simulation dispatches.

### Live simulation progress

Set `CLOCKWORK_PROGRESS_INTERVAL` to a positive wall-clock interval to report exact live drive-loop
counters to standard error from every active cluster:

```powershell
$env:CLOCKWORK_PROGRESS_INTERVAL = "5s"
dotnet run --project tests\Clockwork.Tests\Clockwork.Tests.csproj
```

Test projects which reference `Clockwork.Simulation.Testing` and use Microsoft Testing Platform can
set the same interval when invoking their generated test executable:

```powershell
dotnet run --project tests\Clockwork.Tests\Clockwork.Tests.csproj -- --clockwork-progress 5s
```

Each line includes the runtime id and seed, wall-clock elapsed time, drive-loop iterations, scheduled
steps executed, virtual-time advances, simulated elapsed time, pending scheduler operations, and
runnable/waiting/blocked queue counts. Reporting observes the simulation without scheduling simulated
work, so enabling it does not change deterministic execution.

## Execution results and diagnostics

Every drive method returns a `SimulationExecutionResult` describing exactly why the run stopped:
Expand Down
23 changes: 21 additions & 2 deletions src/Clockwork.Testing/Clockwork.Testing.csproj
Original file line number Diff line number Diff line change
@@ -1,17 +1,36 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<Description>Replay-aware test helpers for deterministic Clockwork scenarios, including in-memory log capture, stable test identity seeds, failure artifacts, and environment-driven replay.</Description>
<Description>Test helpers and Microsoft Testing Platform integration for deterministic Clockwork scenarios, including live progress, in-memory log capture, stable test identity seeds, failure artifacts, and environment-driven replay.</Description>
<RootNamespace>Clockwork.Testing</RootNamespace>
<IsPackable>false</IsPackable>
<PackageId>Clockwork.Simulation.Testing</PackageId>
<Version>0.1.0</Version>
<PackageTags>simulation;testing;distributed-systems;deterministic;microsoft-testing-platform</PackageTags>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<PackageReadmeFile>README.md</PackageReadmeFile>
<RepositoryUrl>https://github.com/ReubenBond/Clockwork</RepositoryUrl>
<IsPackable>true</IsPackable>
<IsTestingPlatformApplication>false</IsTestingPlatformApplication>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
<PackageReference Include="Microsoft.Testing.Platform" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\Clockwork\Clockwork.csproj" />
</ItemGroup>

<ItemGroup>
<None Include="buildTransitive\Clockwork.Simulation.Testing.props"
Pack="true"
PackagePath="buildTransitive\" />
<None Include="buildTransitive\Clockwork.Simulation.Testing.props"
Pack="true"
PackagePath="buildMultiTargeting\" />
<None Include="..\..\README.md" Pack="true" PackagePath="\" />
<InternalsVisibleTo Include="Clockwork.Testing.Tests" />
</ItemGroup>

</Project>
223 changes: 223 additions & 0 deletions src/Clockwork.Testing/ClockworkProgressCommandLineOptionsProvider.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
using System.Globalization;
using System.Text;
using Microsoft.Testing.Platform.Builder;
using Microsoft.Testing.Platform.CommandLine;
using Microsoft.Testing.Platform.Extensions;
using Microsoft.Testing.Platform.Extensions.CommandLine;
using Microsoft.Testing.Platform.Extensions.OutputDevice;
using Microsoft.Testing.Platform.Extensions.TestHost;
using Microsoft.Testing.Platform.Extensions.TestHostControllers;
using Microsoft.Testing.Platform.OutputDevice;
using Microsoft.Testing.Platform.Services;

namespace Clockwork.Testing;

/// <summary>Registers Clockwork's Microsoft Testing Platform command-line options.</summary>
public static class TestingPlatformBuilderHook
{
/// <summary>Adds Clockwork test-runner extensions to the application builder.</summary>
/// <param name="testApplicationBuilder">The test application builder.</param>
/// <param name="_">The test application command-line arguments.</param>
public static void AddExtensions(ITestApplicationBuilder testApplicationBuilder, string[] _)
{
ArgumentNullException.ThrowIfNull(testApplicationBuilder);
testApplicationBuilder.CommandLine.AddProvider(static () => new ClockworkProgressCommandLineOptionsProvider());
testApplicationBuilder.TestHost.AddTestHostApplicationLifetime(
static serviceProvider => new ClockworkProgressOutputLifetime(serviceProvider.GetOutputDevice()));
testApplicationBuilder.TestHostControllers.AddEnvironmentVariableProvider(
static serviceProvider => new ClockworkProgressEnvironmentVariableProvider(
serviceProvider.GetCommandLineOptions()));
}
}

internal sealed class ClockworkProgressCommandLineOptionsProvider : ICommandLineOptionsProvider
{
internal const string OptionName = "clockwork-progress";

private static readonly CommandLineOption[] s_options =
[
new(
OptionName,
"Report live simulation iterations, executed steps, time advances, simulated time, and pending work at this wall-clock interval (for example, 5s).",
ArgumentArity.ExactlyOne,
isHidden: false),
];

public string Uid => "ClockworkProgressCommandLineOptionsProvider";

public string Version => "0.1.0";

public string DisplayName => "Clockwork simulation progress";

public string Description => "Enables periodic progress output from active Clockwork simulation drive loops.";

public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions() => s_options;

public Task<bool> IsEnabledAsync() => Task.FromResult(true);

public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
{
if (commandOption.Name != OptionName)
{
return ValidationResult.ValidTask;
}

return arguments is [var value] && SimulationProgressEnvironment.TryParseInterval(value, out _)
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(
$"--{OptionName} must be followed by a positive duration such as '5s', '500ms', '2m', or '00:00:05'.");
}

public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
{
if (!commandLineOptions.TryGetOptionArgumentList(OptionName, out string[]? arguments))
{
return ValidationResult.ValidTask;
}

if (arguments is not [var value] ||
!SimulationProgressEnvironment.TryParseInterval(value, out TimeSpan interval))
{
return ValidationResult.InvalidTask(
$"--{OptionName} must be followed by a positive duration such as '5s', '500ms', '2m', or '00:00:05'.");
}

Environment.SetEnvironmentVariable(
SimulationProgressEnvironment.Interval,
interval.ToString("c", CultureInfo.InvariantCulture));
return ValidationResult.ValidTask;
}
}

internal sealed class ClockworkProgressEnvironmentVariableProvider : ITestHostEnvironmentVariableProvider
{
private readonly string? _value;
private readonly string? _validationError;

public ClockworkProgressEnvironmentVariableProvider(ICommandLineOptions commandLineOptions)
{
ArgumentNullException.ThrowIfNull(commandLineOptions);

string? value = commandLineOptions.TryGetOptionArgumentList(
ClockworkProgressCommandLineOptionsProvider.OptionName,
out string[]? arguments)
? arguments is [var argument] ? argument : null
: Environment.GetEnvironmentVariable(SimulationProgressEnvironment.Interval);

if (string.IsNullOrWhiteSpace(value))
{
return;
}

_value = value;
if (!SimulationProgressEnvironment.TryParseInterval(value, out _))
{
_validationError =
$"{SimulationProgressEnvironment.Interval} must be a positive duration such as " +
$"'5s', '500ms', '2m', or '00:00:05', not '{value}'.";
}
}

public string Uid => "ClockworkProgressEnvironmentVariableProvider";

public string Version => "0.1.0";

public string DisplayName => "Clockwork simulation progress environment";

public string Description => "Forwards Clockwork progress configuration to orchestrated test hosts.";

public Task<bool> IsEnabledAsync() => Task.FromResult(_value is not null);

public Task UpdateAsync(IEnvironmentVariables environmentVariables)
{
ArgumentNullException.ThrowIfNull(environmentVariables);
if (_value is not null)
{
environmentVariables.SetVariable(new EnvironmentVariable(
SimulationProgressEnvironment.Interval,
_value,
isSecret: false,
isLocked: true));
}

return Task.CompletedTask;
}

public Task<ValidationResult> ValidateTestHostEnvironmentVariablesAsync(
IReadOnlyEnvironmentVariables environmentVariables)
{
ArgumentNullException.ThrowIfNull(environmentVariables);
if (_validationError is not null)
{
return ValidationResult.InvalidTask(_validationError);
}

if (_value is null)
{
return ValidationResult.ValidTask;
}

return environmentVariables.TryGetVariable(
SimulationProgressEnvironment.Interval,
out OwnedEnvironmentVariable? configured) &&
configured.Value == _value
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(
$"Unable to pass {SimulationProgressEnvironment.Interval} to the test host.");
}
}

internal sealed class ClockworkProgressOutputLifetime :
ITestHostApplicationLifetime,
IOutputDeviceDataProducer,
IDisposable
{
private readonly TextWriter _writer;

public ClockworkProgressOutputLifetime(IOutputDevice outputDevice)
{
ArgumentNullException.ThrowIfNull(outputDevice);
_writer = new OutputDeviceTextWriter(outputDevice, this);
}

public string Uid => "ClockworkProgressOutputLifetime";

public string Version => "0.1.0";

public string DisplayName => "Clockwork simulation progress output";

public string Description => "Routes Clockwork progress through the Microsoft Testing Platform output device.";

public Task<bool> IsEnabledAsync() => Task.FromResult(true);

public Task BeforeRunAsync(CancellationToken cancellationToken)
{
SimulationProgressOutput.SetWriter(_writer);
return Task.CompletedTask;
}

public Task AfterRunAsync(int exitCode, CancellationToken cancellationToken)
{
SimulationProgressOutput.SetWriter(null);
return Task.CompletedTask;
}

public void Dispose()
{
SimulationProgressOutput.SetWriter(null);
_writer.Dispose();
}

private sealed class OutputDeviceTextWriter(
IOutputDevice outputDevice,
IOutputDeviceDataProducer producer) : TextWriter
{
public override Encoding Encoding => Encoding.UTF8;

public override void WriteLine(string? value) =>
outputDevice.DisplayAsync(
producer,
new TextOutputDeviceData(value ?? string.Empty),
CancellationToken.None).GetAwaiter().GetResult();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<Project>
<ItemGroup>
<TestingPlatformBuilderHook Include="9F2B15B0-6B0E-4AAF-A66C-DC2CA4D5D01B">
<DisplayName>Clockwork.Simulation.Testing</DisplayName>
<TypeFullName>Clockwork.Testing.TestingPlatformBuilderHook</TypeFullName>
</TestingPlatformBuilderHook>
</ItemGroup>
</Project>
1 change: 1 addition & 0 deletions src/Clockwork/AssemblyInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
[assembly: InternalsVisibleTo("Clockwork.Tests")]
[assembly: InternalsVisibleTo("Clockwork.Runtime.Tests")]
[assembly: InternalsVisibleTo("Clockwork.Benchmarks")]
[assembly: InternalsVisibleTo("Clockwork.Testing")]
14 changes: 12 additions & 2 deletions src/Clockwork/Cluster/SimulationCluster.Adaptive.cs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ public SimulationExecutionResult RunUntil(
ArgumentNullException.ThrowIfNull(budget);
using var control = Scheduler.EnterControlScope();
using var _ = Guard.Enter();
SimulationProgressReporter? progressReporter = SimulationProgressReporter.CreateFromEnvironment(
RuntimeIdentity,
CapturePendingWorkSummary,
() => Scheduler.PendingOperationCount);
return RunAdaptiveCore(
budget,
(batchMaxIterations, consecutiveTimeAdvances) => ExecuteDriveLoop(
Expand All @@ -71,7 +75,8 @@ public SimulationExecutionResult RunUntil(
observeTeardownCancellation: false,
initialConsecutiveTimeAdvances: consecutiveTimeAdvances,
absoluteEndTime: null,
cancellationToken: cancellationToken));
cancellationToken: cancellationToken,
progressReporter: progressReporter));
}

/// <summary>
Expand Down Expand Up @@ -103,6 +108,10 @@ public SimulationExecutionResult RunUntilIdle(
ArgumentNullException.ThrowIfNull(budget);
using var control = Scheduler.EnterControlScope();
using var _ = Guard.Enter();
SimulationProgressReporter? progressReporter = SimulationProgressReporter.CreateFromEnvironment(
RuntimeIdentity,
CapturePendingWorkSummary,
() => Scheduler.PendingOperationCount);
return RunAdaptiveCore(
budget,
(batchMaxIterations, consecutiveTimeAdvances) => ExecuteDriveLoop(
Expand All @@ -112,7 +121,8 @@ public SimulationExecutionResult RunUntilIdle(
observeTeardownCancellation: true,
initialConsecutiveTimeAdvances: consecutiveTimeAdvances,
absoluteEndTime: null,
cancellationToken: cancellationToken));
cancellationToken: cancellationToken,
progressReporter: progressReporter));
}
#pragma warning restore CA1068

Expand Down
Loading