From 1aa656f86fd9c4e585c44535156c357c17a76282 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Sat, 20 Apr 2024 23:54:23 -0700 Subject: [PATCH 01/28] Non-reentrant timers --- .../Storage/AzureStoragePolicyOptions.cs | 3 +- .../Core/Grain.Timers.cs | 142 ++++++ src/Orleans.Core.Abstractions/Core/Grain.cs | 474 +++++++++--------- .../Core/IGrainBase.cs | 50 ++ .../Core/IGrainContext.cs | 35 +- .../Runtime/IGrainRuntime.cs | 110 ++-- .../Runtime/IGrainTimer.cs | 3 +- .../Timers/GrainTimerCreationOptions.cs | 69 +++ .../Timers/ITimerRegistry.cs | 8 +- .../Async/AsyncExecutorWithRetries.cs | 3 +- .../Async/MultiTaskCompletionSource.cs | 81 ++- src/Orleans.Core/Messaging/Message.cs | 42 +- src/Orleans.Core/Networking/Connection.cs | 1 + .../Providers/GrainStorageHelpers.cs | 68 ++- .../Runtime/AsyncEnumerableGrainExtension.cs | 17 +- .../Runtime/ClientGrainContext.cs | 6 +- src/Orleans.Core/Runtime/Constants.cs | 39 +- .../Runtime/InvokableObjectManager.cs | 6 +- .../Runtime/OutsideRuntimeClient.cs | 2 +- .../Catalog/ActivationCollector.cs | 28 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 290 ++++++----- src/Orleans.Runtime/Catalog/Catalog.cs | 15 +- .../Catalog/StatelessWorkerGrainContext.cs | 61 +-- src/Orleans.Runtime/Core/GrainRuntime.cs | 145 +++--- src/Orleans.Runtime/Core/HostedClient.cs | 22 +- .../Core/InsideRuntimeClient.cs | 2 +- .../Core/InternalGrainRuntime.cs | 54 +- src/Orleans.Runtime/Core/SystemTarget.cs | 55 +- .../Hosting/DefaultSiloServices.cs | 1 - .../Messaging/MessageCenter.cs | 21 +- .../Placement/PlacementService.cs | 18 +- src/Orleans.Runtime/Silo/Silo.cs | 11 +- src/Orleans.Runtime/Timers/GrainTimer.cs | 347 ++++++++++--- .../Timers/IGrainTimerRegistry.cs | 25 + src/Orleans.Runtime/Timers/TimerRegistry.cs | 28 +- .../Invocation/ITargetHolder.cs | 34 +- .../PersistentStreamPullingAgent.cs | 44 +- test/DefaultCluster.Tests/TimerOrleansTest.cs | 56 +++ .../Streaming/EHProgrammaticSubscribeTests.cs | 2 +- .../Streaming/AQProgrammaticSubscribeTest.cs | 2 +- .../Grains/TestGrainInterfaces/ITimerGrain.cs | 13 + test/Grains/TestGrains/GenericGrains.cs | 8 +- test/Grains/TestGrains/LivenessTestGrain.cs | 4 +- .../TypedProducerGrain.cs | 4 +- .../Grains/TestGrains/SampleStreamingGrain.cs | 6 +- .../Grains/TestGrains/StatelessWorkerGrain.cs | 8 +- test/Grains/TestGrains/StuckGrain.cs | 1 - .../TestInternalGrains/CollectionTestGrain.cs | 6 +- .../TestInternalGrains/StreamingGrain.cs | 4 +- test/Grains/TestInternalGrains/TestGrain.cs | 5 +- test/Grains/TestInternalGrains/TimerGrain.cs | 294 ++++++++++- .../Async_AsyncExecutorWithRetriesTests.cs | 2 +- .../OrleansTaskSchedulerBasicTests.cs | 6 +- test/Tester/Forwarding/ShutdownSiloTests.cs | 1 - test/Tester/GrainCallFilterTests.cs | 3 +- .../MemoryProgrammaticSubcribeTests.cs | 2 +- ...cs => ProgrammaticSubscribeTestsRunner.cs} | 4 +- ...serverWithImplicitSubscribingTestRunner.cs | 6 +- .../ActivationCollectorTests.cs | 21 + .../DeactivateOnIdleTests.cs | 2 - 60 files changed, 1817 insertions(+), 1003 deletions(-) create mode 100644 src/Orleans.Core.Abstractions/Core/Grain.Timers.cs create mode 100644 src/Orleans.Core.Abstractions/Timers/GrainTimerCreationOptions.cs create mode 100644 src/Orleans.Runtime/Timers/IGrainTimerRegistry.cs rename test/Tester/StreamingTests/ProgrammaticSubscribeTests/{ProgrammaticSubcribeTestsRunner.cs => ProgrammaticSubscribeTestsRunner.cs} (99%) diff --git a/src/Azure/Shared/Storage/AzureStoragePolicyOptions.cs b/src/Azure/Shared/Storage/AzureStoragePolicyOptions.cs index e1a5d46dbd..03d1cc3147 100644 --- a/src/Azure/Shared/Storage/AzureStoragePolicyOptions.cs +++ b/src/Azure/Shared/Storage/AzureStoragePolicyOptions.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; #if ORLEANS_CLUSTERING namespace Orleans.Clustering.AzureStorage @@ -47,7 +48,7 @@ public TimeSpan OperationTimeout private static void SetIfValidTimeout(ref TimeSpan? field, TimeSpan value, string propertyName) { - if (value > TimeSpan.Zero || value.Equals(TimeSpan.FromMilliseconds(-1))) + if (value > TimeSpan.Zero || value.Equals(Timeout.InfiniteTimeSpan)) { field = value; } diff --git a/src/Orleans.Core.Abstractions/Core/Grain.Timers.cs b/src/Orleans.Core.Abstractions/Core/Grain.Timers.cs new file mode 100644 index 0000000000..f9bcc05962 --- /dev/null +++ b/src/Orleans.Core.Abstractions/Core/Grain.Timers.cs @@ -0,0 +1,142 @@ +#nullable enable +using System; +using System.Threading; +using System.Threading.Tasks; +using Orleans.Runtime; + +namespace Orleans; + +public abstract partial class Grain +{ + /// + /// Creates a grain timer. + /// + /// The timer callback, which will be invoked whenever the timer becomes due. + /// + /// The options for creating the timer. + /// + /// + /// The instance which represents the timer. + /// + /// + /// + /// Grain timers do not keep grains active by default. Setting to + /// causes each timer tick to extend the grain activation's lifetime. + /// If the timer ticks are infrequent, the grain can still be deactivated due to idleness. + /// When a grain is deactivated, all active timers are discarded. + /// + /// + /// Until the returned from the callback is resolved, the next timer tick will not be scheduled. + /// That is to say, a timer callback will never be concurrently executed with itself. + /// If is set to , the timer callback will be allowed + /// to interleave with with other grain method calls and other timers. + /// If is set to , the timer callback will respect the + /// reentrancy setting of the grain, just like a typical grain method call. + /// + /// + /// The timer may be stopped at any time by calling the 's method. + /// Disposing a timer prevents any further timer ticks from being scheduled. + /// + /// + /// The timer due time and period can be updated by calling its method. + /// Each time the timer is updated, the next timer tick will be scheduled based on the updated due time. + /// Subsequent ticks will be scheduled after the updated period elapses. + /// Note that this behavior is the same as the method. + /// + /// + /// Exceptions thrown from the callback will be logged, but will not prevent the next timer tick from being queued. + /// + /// + protected IGrainTimer RegisterGrainTimer(Func callback, GrainTimerCreationOptions options) + { + ArgumentNullException.ThrowIfNull(callback); + return RegisterGrainTimer(static (callback, cancellationToken) => callback(cancellationToken), callback, options); + } + + /// + /// The state passed to the callback. + protected internal IGrainTimer RegisterGrainTimer(Func callback, TState state, GrainTimerCreationOptions options) + { + ArgumentNullException.ThrowIfNull(callback); + + EnsureRuntime(); + return Runtime.TimerRegistry.RegisterGrainTimer(GrainContext, callback, state, options); + } + + /// + protected IGrainTimer RegisterGrainTimer(Func callback, GrainTimerCreationOptions options) + { + ArgumentNullException.ThrowIfNull(callback); + return RegisterGrainTimer(static (callback, cancellationToken) => callback(), callback, options); + } + + /// + /// The state passed to the callback. + protected IGrainTimer RegisterGrainTimer(Func callback, TState state, GrainTimerCreationOptions options) + { + ArgumentNullException.ThrowIfNull(callback); + return RegisterGrainTimer(static (state, _) => state.Callback(state.State), (Callback: callback, State: state), options); + } + + /// + /// Creates a grain timer. + /// + /// The timer callback, which will be invoked whenever the timer becomes due. + /// + /// A representing the amount of time to delay before invoking the callback method specified when the was constructed. + /// Specify to prevent the timer from starting. + /// Specify to start the timer immediately. + /// + /// + /// The time interval between invocations of the callback method specified when the was constructed. + /// Specify to disable periodic signaling. + /// + /// + /// The instance which represents the timer. + /// + /// + /// + /// Grain timers do not keep grains active by default. Setting to + /// causes each timer tick to extend the grain activation's lifetime. + /// If the timer ticks are infrequent, the grain can still be deactivated due to idleness. + /// When a grain is deactivated, all active timers are discarded. + /// + /// + /// Until the returned from the callback is resolved, the next timer tick will not be scheduled. + /// That is to say, a timer callback will never be concurrently executed with itself. + /// If is set to , the timer callback will be allowed + /// to interleave with with other grain method calls and other timers. + /// If is set to , the timer callback will respect the + /// reentrancy setting of the grain, just like a typical grain method call. + /// + /// + /// The timer may be stopped at any time by calling the 's method. + /// Disposing a timer prevents any further timer ticks from being scheduled. + /// + /// + /// The timer due time and period can be updated by calling its method. + /// Each time the timer is updated, the next timer tick will be scheduled based on the updated due time. + /// Subsequent ticks will be scheduled after the updated period elapses. + /// Note that this behavior is the same as the method. + /// + /// + /// Exceptions thrown from the callback will be logged, but will not prevent the next timer tick from being queued. + /// + /// + protected IGrainTimer RegisterGrainTimer(Func callback, TimeSpan dueTime, TimeSpan period) + => RegisterGrainTimer(callback, new() { DueTime = dueTime, Period = period }); + + /// + protected IGrainTimer RegisterGrainTimer(Func callback, TimeSpan dueTime, TimeSpan period) + => RegisterGrainTimer(callback, new() { DueTime = dueTime, Period = period }); + + /// + /// The state passed to the callback. + protected IGrainTimer RegisterGrainTimer(Func callback, TState state, TimeSpan dueTime, TimeSpan period) + => RegisterGrainTimer(callback, state, new() { DueTime = dueTime, Period = period }); + + /// + /// The state passed to the callback. + protected IGrainTimer RegisterGrainTimer(Func callback, TState state, TimeSpan dueTime, TimeSpan period) + => RegisterGrainTimer(callback, state, new() { DueTime = dueTime, Period = period }); +} diff --git a/src/Orleans.Core.Abstractions/Core/Grain.cs b/src/Orleans.Core.Abstractions/Core/Grain.cs index 4da1b9b0eb..a6a5b7156d 100644 --- a/src/Orleans.Core.Abstractions/Core/Grain.cs +++ b/src/Orleans.Core.Abstractions/Core/Grain.cs @@ -7,274 +7,272 @@ using Orleans.Runtime; using Orleans.Serialization.TypeSystem; -namespace Orleans +namespace Orleans; + +/// +/// The abstract base class for all grain classes. +/// +public abstract partial class Grain : IGrainBase, IAddressable { + // Do not use this directly because we currently don't provide a way to inject it; + // any interaction with it will result in non unit-testable code. Any behavior that can be accessed + // from within client code (including subclasses of this class), should be exposed through IGrainRuntime. + // The better solution is to refactor this interface and make it injectable through the constructor. + internal IGrainContext GrainContext { get; private set; } + + IGrainContext IGrainBase.GrainContext => GrainContext; + + public GrainReference GrainReference { get { return GrainContext.GrainReference; } } + + internal IGrainRuntime Runtime { get; } + + /// + /// Gets an object which can be used to access other grains. Null if this grain is not associated with a Runtime, such as when created directly for unit testing. + /// + protected IGrainFactory GrainFactory => Runtime.GrainFactory; + + /// + /// Gets the IServiceProvider managed by the runtime. Null if this grain is not associated with a Runtime, such as when created directly for unit testing. + /// + protected internal IServiceProvider ServiceProvider => GrainContext?.ActivationServices ?? Runtime?.ServiceProvider!; + + internal GrainId GrainId => GrainContext.GrainId; + /// - /// The abstract base class for all grain classes. + /// This constructor should never be invoked. We expose it so that client code (subclasses of Grain) do not have to add a constructor. + /// Client code should use the GrainFactory property to get a reference to a Grain. /// - public abstract class Grain : IGrainBase, IAddressable + protected Grain() : this(RuntimeContext.Current, grainRuntime: null) + {} + + /// + /// Grain implementers do NOT have to expose this constructor but can choose to do so. + /// This constructor is particularly useful for unit testing where test code can create a Grain and replace + /// the IGrainIdentity and IGrainRuntime with test doubles (mocks/stubs). + /// + protected Grain(IGrainContext grainContext, IGrainRuntime? grainRuntime = null) { - // Do not use this directly because we currently don't provide a way to inject it; - // any interaction with it will result in non unit-testable code. Any behaviour that can be accessed - // from within client code (including subclasses of this class), should be exposed through IGrainRuntime. - // The better solution is to refactor this interface and make it injectable through the constructor. - internal IGrainContext GrainContext { get; private set; } - - IGrainContext IGrainBase.GrainContext => GrainContext; - - public GrainReference GrainReference { get { return GrainContext.GrainReference; } } - - internal IGrainRuntime Runtime { get; } - - /// - /// Gets an object which can be used to access other grains. Null if this grain is not associated with a Runtime, such as when created directly for unit testing. - /// - protected IGrainFactory GrainFactory => Runtime.GrainFactory; - - /// - /// Gets the IServiceProvider managed by the runtime. Null if this grain is not associated with a Runtime, such as when created directly for unit testing. - /// - protected internal IServiceProvider ServiceProvider => GrainContext?.ActivationServices ?? Runtime?.ServiceProvider!; - - internal GrainId GrainId => GrainContext.GrainId; - - /// - /// This constructor should never be invoked. We expose it so that client code (subclasses of Grain) do not have to add a constructor. - /// Client code should use the GrainFactory property to get a reference to a Grain. - /// - protected Grain() : this(RuntimeContext.Current, grainRuntime: null) - {} - - /// - /// Grain implementers do NOT have to expose this constructor but can choose to do so. - /// This constructor is particularly useful for unit testing where test code can create a Grain and replace - /// the IGrainIdentity and IGrainRuntime with test doubles (mocks/stubs). - /// - protected Grain(IGrainContext grainContext, IGrainRuntime? grainRuntime = null) - { - GrainContext = grainContext; - Runtime = grainRuntime ?? grainContext?.ActivationServices.GetService()!; - } + GrainContext = grainContext; + Runtime = grainRuntime ?? grainContext?.ActivationServices.GetService()!; + } - /// - /// String representation of grain's SiloIdentity including type and primary key. - /// - public string IdentityString => GrainId.ToString(); - - /// - /// A unique identifier for the current silo. - /// There is no semantic content to this string, but it may be useful for logging. - /// - public string RuntimeIdentity => Runtime?.SiloIdentity ?? string.Empty; - - /// - /// Registers a timer to send periodic callbacks to this grain. - /// - /// - /// - /// This timer will not prevent the current grain from being deactivated. - /// If the grain is deactivated, then the timer will be discarded. - /// - /// - /// Until the Task returned from the asyncCallback is resolved, - /// the next timer tick will not be scheduled. - /// That is to say, timer callbacks never interleave their turns. - /// - /// - /// The timer may be stopped at any time by calling the Dispose method - /// on the timer handle returned from this call. - /// - /// - /// Any exceptions thrown by or faulted Task's returned from the asyncCallback - /// will be logged, but will not prevent the next timer tick from being queued. - /// - /// - /// Callback function to be invoked when timer ticks. - /// State object that will be passed as argument when calling the asyncCallback. - /// Due time for first timer tick. - /// Period of subsequent timer ticks. - /// Handle for this Timer. - /// - protected IDisposable RegisterTimer(Func asyncCallback, object? state, TimeSpan dueTime, TimeSpan period) - { - if (asyncCallback == null) - throw new ArgumentNullException(nameof(asyncCallback)); + /// + /// String representation of grain's SiloIdentity including type and primary key. + /// + public string IdentityString => GrainId.ToString(); - EnsureRuntime(); - return Runtime.TimerRegistry.RegisterTimer(GrainContext ?? RuntimeContext.Current, asyncCallback, state, dueTime, period); - } + /// + /// A unique identifier for the current silo. + /// There is no semantic content to this string, but it may be useful for logging. + /// + public string RuntimeIdentity => Runtime?.SiloIdentity ?? string.Empty; - /// - /// Deactivate this activation of the grain after the current grain method call is completed. - /// This call will mark this activation of the current grain to be deactivated and removed at the end of the current method. - /// The next call to this grain will result in a different activation to be used, which typical means a new activation will be created automatically by the runtime. - /// - protected void DeactivateOnIdle() - { - EnsureRuntime(); - Runtime.DeactivateOnIdle(GrainContext ?? RuntimeContext.Current); - } + /// + /// Registers a timer to send periodic callbacks to this grain. + /// + /// + /// + /// This timer will not prevent the current grain from being deactivated. + /// If the grain is deactivated, then the timer will be discarded. + /// + /// + /// Until the Task returned from the callback is resolved, + /// the next timer tick will not be scheduled. + /// That is to say, timer callbacks never interleave their turns. + /// + /// + /// The timer may be stopped at any time by calling the Dispose method + /// on the timer handle returned from this call. + /// + /// + /// Any exceptions thrown by or faulted Task's returned from the callback + /// will be logged, but will not prevent the next timer tick from being queued. + /// + /// + /// Callback function to be invoked when timer ticks. + /// State object that will be passed as argument when calling the . + /// Due time for first timer tick. + /// Period of subsequent timer ticks. + /// Handle for this timer. + [Obsolete("Use 'RegisterGrainTimer(callback, state, new() { DueTime = dueTime, Period = period, Interleave = true })' instead.")] + protected IDisposable RegisterTimer(Func callback, object? state, TimeSpan dueTime, TimeSpan period) + { + ArgumentNullException.ThrowIfNull(callback); - /// - /// Starts an attempt to migrating this instance to another location. - /// Migration captures the current , making it available to the activation's placement director so that it can consider it when selecting a new location. - /// Migration will occur asynchronously, when no requests are executing, and will not occur if the activation's placement director does not select an alternative location. - /// - protected void MigrateOnIdle() - { - EnsureRuntime(); - ((IGrainBase)this).MigrateOnIdle(); - } + EnsureRuntime(); + return Runtime.TimerRegistry.RegisterTimer(GrainContext ?? RuntimeContext.Current, callback, state, dueTime, period); + } - /// - /// Delay Deactivation of this activation at least for the specified time duration. - /// A positive timeSpan value means “prevent GC of this activation for that time span”. - /// A negative timeSpan value means “cancel the previous setting of the DelayDeactivation call and make this activation behave based on the regular Activation Garbage Collection settings”. - /// DeactivateOnIdle method would undo / override any current “keep alive” setting, - /// making this grain immediately available for deactivation. - /// - protected void DelayDeactivation(TimeSpan timeSpan) - { - EnsureRuntime(); - Runtime.DelayDeactivation(GrainContext ?? RuntimeContext.Current, timeSpan); - } + /// + /// Deactivate this activation of the grain after the current grain method call is completed. + /// This call will mark this activation of the current grain to be deactivated and removed at the end of the current method. + /// The next call to this grain will result in a different activation to be used, which typical means a new activation will be created automatically by the runtime. + /// + protected void DeactivateOnIdle() + { + EnsureRuntime(); + Runtime.DeactivateOnIdle(GrainContext ?? RuntimeContext.Current); + } - /// - /// This method is called at the end of the process of activating a grain. - /// It is called before any messages have been dispatched to the grain. - /// For grains with declared persistent state, this method is called after the State property has been populated. - /// - /// A cancellation token which signals when activation is being canceled. - public virtual Task OnActivateAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - /// - /// This method is called at the beginning of the process of deactivating a grain. - /// - /// The reason for deactivation. Informational only. - /// A cancellation token which signals when deactivation should complete promptly. - public virtual Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) => Task.CompletedTask; - - private void EnsureRuntime() - { - if (Runtime == null) - { - throw new InvalidOperationException("Grain was created outside of the Orleans creation process and no runtime was specified."); - } - } + /// + /// Starts an attempt to migrating this instance to another location. + /// Migration captures the current , making it available to the activation's placement director so that it can consider it when selecting a new location. + /// Migration will occur asynchronously, when no requests are executing, and will not occur if the activation's placement director does not select an alternative location. + /// + protected void MigrateOnIdle() + { + EnsureRuntime(); + ((IGrainBase)this).MigrateOnIdle(); } /// - /// Base class for a Grain with declared persistent state. + /// Delay Deactivation of this activation at least for the specified time duration. + /// A positive timeSpan value means “prevent GC of this activation for that time span”. + /// A negative timeSpan value means “cancel the previous setting of the DelayDeactivation call and make this activation behave based on the regular Activation Garbage Collection settings”. + /// DeactivateOnIdle method would undo / override any current “keep alive” setting, + /// making this grain immediately available for deactivation. /// - /// The class of the persistent state object - public class Grain : Grain + protected void DelayDeactivation(TimeSpan timeSpan) { - /// - /// The underlying state storage. - /// - private IStorage? _storage; - - /// - /// Initializes a new instance of the class. - /// - /// - /// This constructor should never be invoked. We expose it so that client code (subclasses of this class) do not have to add a constructor. - /// Client code should use the GrainFactory to get a reference to a Grain. - /// - protected Grain() - { - var observer = new LifecycleObserver(this); - var lifecycle = RuntimeContext.Current.ObservableLifecycle; - lifecycle.AddMigrationParticipant(observer); - lifecycle.Subscribe(RuntimeTypeNameFormatter.Format(GetType()), GrainLifecycleStage.SetupState, observer); - } + EnsureRuntime(); + Runtime.DelayDeactivation(GrainContext ?? RuntimeContext.Current, timeSpan); + } - /// - /// Initializes a new instance of the class. - /// - /// - /// The storage implementation. - /// - /// - /// Grain implementers do NOT have to expose this constructor but can choose to do so. - /// This constructor is particularly useful for unit testing where test code can create a Grain and replace - /// the IGrainIdentity, IGrainRuntime and State with test doubles (mocks/stubs). - /// - protected Grain(IStorage storage) - { - _storage = storage; - } + /// + /// This method is called at the end of the process of activating a grain. + /// It is called before any messages have been dispatched to the grain. + /// For grains with declared persistent state, this method is called after the State property has been populated. + /// + /// A cancellation token which signals when activation is being canceled. + public virtual Task OnActivateAsync(CancellationToken cancellationToken) => Task.CompletedTask; - /// - /// Gets or sets the grain state. - /// - protected TGrainState State + /// + /// This method is called at the beginning of the process of deactivating a grain. + /// + /// The reason for deactivation. Informational only. + /// A cancellation token which signals when deactivation should complete promptly. + public virtual Task OnDeactivateAsync(DeactivationReason reason, CancellationToken cancellationToken) => Task.CompletedTask; + + private void EnsureRuntime() + { + if (Runtime == null) { - get => _storage!.State; - set => _storage!.State = value; + throw new InvalidOperationException("Grain was created outside of the Orleans creation process and no runtime was specified."); } + } +} - /// - /// Clears the current grain state data from backing store. - /// - /// - /// A representing the operation. - /// - protected virtual Task ClearStateAsync() => _storage!.ClearStateAsync(); - - /// - /// Write the current grain state data into the backing store. - /// - /// - /// A representing the operation. - /// - protected virtual Task WriteStateAsync() => _storage!.WriteStateAsync(); - - /// - /// Reads grain state from backing store, updating . - /// - /// - /// Any previous contents of the grain state data will be overwritten. - /// - /// - /// A representing the operation. - /// - protected virtual Task ReadStateAsync() => _storage!.ReadStateAsync(); - - private class LifecycleObserver : ILifecycleObserver, IGrainMigrationParticipant - { - private readonly Grain _grain; +/// +/// Base class for a Grain with declared persistent state. +/// +/// The class of the persistent state object +public class Grain : Grain +{ + /// + /// The underlying state storage. + /// + private IStorage? _storage; + + /// + /// Initializes a new instance of the class. + /// + /// + /// This constructor should never be invoked. We expose it so that client code (subclasses of this class) do not have to add a constructor. + /// Client code should use the GrainFactory to get a reference to a Grain. + /// + protected Grain() + { + var observer = new LifecycleObserver(this); + var lifecycle = RuntimeContext.Current.ObservableLifecycle; + lifecycle.AddMigrationParticipant(observer); + lifecycle.Subscribe(RuntimeTypeNameFormatter.Format(GetType()), GrainLifecycleStage.SetupState, observer); + } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The storage implementation. + /// + /// + /// Grain implementers do NOT have to expose this constructor but can choose to do so. + /// This constructor is particularly useful for unit testing where test code can create a Grain and replace + /// the IGrainIdentity, IGrainRuntime and State with test doubles (mocks/stubs). + /// + protected Grain(IStorage storage) + { + _storage = storage; + } - public LifecycleObserver(Grain grain) => _grain = grain; + /// + /// Gets or sets the grain state. + /// + protected TGrainState State + { + get => _storage!.State; + set => _storage!.State = value; + } - private void SetupStorage() => _grain._storage ??= _grain.Runtime.GetStorage(_grain.GrainContext); + /// + /// Clears the current grain state data from backing store. + /// + /// + /// A representing the operation. + /// + protected virtual Task ClearStateAsync() => _storage!.ClearStateAsync(); - public void OnDehydrate(IDehydrationContext dehydrationContext) => (_grain._storage as IGrainMigrationParticipant)?.OnDehydrate(dehydrationContext); + /// + /// Write the current grain state data into the backing store. + /// + /// + /// A representing the operation. + /// + protected virtual Task WriteStateAsync() => _storage!.WriteStateAsync(); - public void OnRehydrate(IRehydrationContext rehydrationContext) + /// + /// Reads grain state from backing store, updating . + /// + /// + /// Any previous contents of the grain state data will be overwritten. + /// + /// + /// A representing the operation. + /// + protected virtual Task ReadStateAsync() => _storage!.ReadStateAsync(); + + private class LifecycleObserver : ILifecycleObserver, IGrainMigrationParticipant + { + private readonly Grain _grain; + + public LifecycleObserver(Grain grain) => _grain = grain; + + private void SetupStorage() => _grain._storage ??= _grain.Runtime.GetStorage(_grain.GrainContext); + + public void OnDehydrate(IDehydrationContext dehydrationContext) => (_grain._storage as IGrainMigrationParticipant)?.OnDehydrate(dehydrationContext); + + public void OnRehydrate(IRehydrationContext rehydrationContext) + { + SetupStorage(); + (_grain._storage as IGrainMigrationParticipant)?.OnRehydrate(rehydrationContext); + } + + public Task OnStart(CancellationToken cancellationToken = default) + { + if (cancellationToken.IsCancellationRequested) { - SetupStorage(); - (_grain._storage as IGrainMigrationParticipant)?.OnRehydrate(rehydrationContext); + return Task.CompletedTask; } - public Task OnStart(CancellationToken cancellationToken = default) + // Avoid reading the state if it is already present because of rehydration + if (_grain._storage?.Etag is not null) { - if (cancellationToken.IsCancellationRequested) - { - return Task.CompletedTask; - } - - // Avoid reading the state if it is already present because of rehydration - if (_grain._storage?.Etag is not null) - { - return Task.CompletedTask; - } - - SetupStorage(); - return _grain.ReadStateAsync(); + return Task.CompletedTask; } - public Task OnStop(CancellationToken cancellationToken = default) => Task.CompletedTask; + SetupStorage(); + return _grain.ReadStateAsync(); } + + public Task OnStop(CancellationToken cancellationToken = default) => Task.CompletedTask; } } diff --git a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs index bbdabee371..7c95ee83c0 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainBase.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainBase.cs @@ -1,6 +1,9 @@ +using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; using Orleans.Runtime; +using Orleans.Timers; namespace Orleans { @@ -49,6 +52,53 @@ public static class GrainBaseExtensions /// Migration will occur asynchronously, when no requests are executing, and will not occur if the activation's placement director does not select an alternative location. /// public static void MigrateOnIdle(this IGrainBase grain) => grain.GrainContext.Migrate(RequestContext.CallContextData?.Value.Values); + + /// + /// Creates a grain timer. + /// + /// The timer callback, which will be invoked whenever the timer becomes due. + /// The state passed to the callback. + /// + /// The options for creating the timer. + /// + /// + /// The instance which represents the timer. + /// + /// + /// + /// Grain timers do not keep grains active by default. Setting to + /// causes each timer tick to extend the grain activation's lifetime. + /// If the timer ticks are infrequent, the grain can still be deactivated due to idleness. + /// When a grain is deactivated, all active timers are discarded. + /// + /// + /// Until the returned from the callback is resolved, the next timer tick will not be scheduled. + /// That is to say, a timer callback will never be concurrently executed with itself. + /// If is set to , the timer callback will be allowed + /// to interleave with with other grain method calls and other timers. + /// If is set to , the timer callback will respect the + /// reentrancy setting of the grain, just like a typical grain method call. + /// + /// + /// The timer may be stopped at any time by calling the 's method. + /// Disposing a timer prevents any further timer ticks from being scheduled. + /// + /// + /// The timer due time and period can be updated by calling its method. + /// Each time the timer is updated, the next timer tick will be scheduled based on the updated due time. + /// Subsequent ticks will be scheduled after the updated period elapses. + /// Note that this behavior is the same as the method. + /// + /// + /// Exceptions thrown from the callback will be logged, but will not prevent the next timer tick from being queued. + /// + /// + public static IGrainTimer RegisterGrainTimer(this IGrainBase grain, Func callback, TState state, GrainTimerCreationOptions options) + { + ArgumentNullException.ThrowIfNull(callback); + if (grain is Grain grainClass) return grainClass.RegisterGrainTimer(callback, state, options); + return grain.GrainContext.ActivationServices.GetRequiredService().RegisterGrainTimer(grain.GrainContext, callback, state, options); + } } /// diff --git a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs index 2b263629cb..0b802e3ca4 100644 --- a/src/Orleans.Core.Abstractions/Core/IGrainContext.cs +++ b/src/Orleans.Core.Abstractions/Core/IGrainContext.cs @@ -1,3 +1,4 @@ +#nullable enable using System; using System.Collections.Generic; using System.Threading; @@ -24,7 +25,7 @@ public interface IGrainContext : ITargetHolder, IEquatable /// /// Gets the grain instance, or if the grain instance has not been set yet. /// - object GrainInstance { get; } + object? GrainInstance { get; } /// /// Gets the activation id. @@ -61,7 +62,7 @@ public interface IGrainContext : ITargetHolder, IEquatable /// /// The type used to lookup this component. /// The component instance. - void SetComponent(TComponent value) where TComponent : class; + void SetComponent(TComponent? value) where TComponent : class; /// /// Submits an incoming message to this instance. @@ -74,14 +75,14 @@ public interface IGrainContext : ITargetHolder, IEquatable /// /// The request context of the request which is causing this instance to be activated, if any. /// A cancellation token which, when canceled, indicates that the process should complete promptly. - void Activate(Dictionary requestContext, CancellationToken? cancellationToken = default); + void Activate(Dictionary? requestContext, CancellationToken cancellationToken = default); /// /// Start deactivating this instance. /// /// The reason for deactivation, for informational purposes. /// A cancellation token which, when canceled, indicates that the process should complete promptly. - void Deactivate(DeactivationReason deactivationReason, CancellationToken? cancellationToken = default); + void Deactivate(DeactivationReason deactivationReason, CancellationToken cancellationToken = default); /// /// Start rehydrating this instance from the provided rehydration context. @@ -95,7 +96,7 @@ public interface IGrainContext : ITargetHolder, IEquatable /// /// The request context, which is provided to the placement director so that it can be examined when selecting a new location. /// A cancellation token which, when canceled, indicates that the process should complete promptly. - void Migrate(Dictionary requestContext, CancellationToken? cancellationToken = default); + void Migrate(Dictionary? requestContext, CancellationToken cancellationToken = default); } /// @@ -116,7 +117,7 @@ public static class GrainContextExtensions /// /// A which will complete once the grain has deactivated. /// - public static Task DeactivateAsync(this IGrainContext grainContext, DeactivationReason deactivationReason, CancellationToken? cancellationToken = default) + public static Task DeactivateAsync(this IGrainContext grainContext, DeactivationReason deactivationReason, CancellationToken cancellationToken = default) { grainContext.Deactivate(deactivationReason, cancellationToken); return grainContext.Deactivated; @@ -179,28 +180,6 @@ internal interface ICollectibleGrainContext : IGrainContext void DelayDeactivation(TimeSpan timeSpan); } - /// - /// Provides functionality to record the creation and deletion of grain timers. - /// - internal interface IGrainTimerRegistry - { - /// - /// Signals to the registry that a timer was created. - /// - /// - /// The timer. - /// - void OnTimerCreated(IGrainTimer timer); - - /// - /// Signals to the registry that a timer was disposed. - /// - /// - /// The timer. - /// - void OnTimerDisposed(IGrainTimer timer); - } - /// /// Functionality to schedule tasks on a grain. /// diff --git a/src/Orleans.Core.Abstractions/Runtime/IGrainRuntime.cs b/src/Orleans.Core.Abstractions/Runtime/IGrainRuntime.cs index dbdbcf9f84..077c9fbd2b 100644 --- a/src/Orleans.Core.Abstractions/Runtime/IGrainRuntime.cs +++ b/src/Orleans.Core.Abstractions/Runtime/IGrainRuntime.cs @@ -1,60 +1,64 @@ +#nullable enable using System; using Orleans.Core; using Orleans.Timers; -namespace Orleans.Runtime +namespace Orleans.Runtime; + +/// +/// The gateway of the to the Orleans runtime. The should only interact with the runtime through this interface. +/// +public interface IGrainRuntime { /// - /// The gateway of the to the Orleans runtime. The should only interact with the runtime through this interface. - /// - public interface IGrainRuntime - { - /// - /// Gets a unique identifier for the current silo. - /// There is no semantic content to this string, but it may be useful for logging. - /// - string SiloIdentity { get; } - - /// - /// Gets the silo address associated with this instance. - /// - SiloAddress SiloAddress { get; } - - /// - /// Gets the grain factory. - /// - IGrainFactory GrainFactory { get; } - - /// - /// Gets the timer registry. - /// - ITimerRegistry TimerRegistry { get; } - - - /// - /// Gets the service provider. - /// - IServiceProvider ServiceProvider { get; } - - /// - /// Deactivates the provided grain when it becomes idle. - /// - /// The grain context. - void DeactivateOnIdle(IGrainContext grainContext); - - /// - /// Delays idle activation collection of the provided grain due to inactivity until at least the specified time has elapsed. - /// - /// The grain context. - /// The time to delay idle activation collection for. - void DelayDeactivation(IGrainContext grainContext, TimeSpan timeSpan); - - /// - /// Gets grain storage for the provided grain. - /// - /// The grain state type. - /// The grain context. - /// The grain storage for the provided grain. - IStorage GetStorage(IGrainContext grainContext); - } + /// Gets a unique identifier for the current silo. + /// There is no semantic content to this string, but it may be useful for logging. + /// + string SiloIdentity { get; } + + /// + /// Gets the silo address associated with this instance. + /// + SiloAddress SiloAddress { get; } + + /// + /// Gets the grain factory. + /// + IGrainFactory GrainFactory { get; } + + /// + /// Gets the timer registry. + /// + ITimerRegistry TimerRegistry { get; } + + /// + /// Gets the service provider. + /// + IServiceProvider ServiceProvider { get; } + + /// + /// Gets the time provider. + /// + TimeProvider TimeProvider => TimeProvider.System; + + /// + /// Deactivates the provided grain when it becomes idle. + /// + /// The grain context. + void DeactivateOnIdle(IGrainContext grainContext); + + /// + /// Delays idle activation collection of the provided grain due to inactivity until at least the specified time has elapsed. + /// + /// The grain context. + /// The time to delay idle activation collection for. + void DelayDeactivation(IGrainContext grainContext, TimeSpan timeSpan); + + /// + /// Gets grain storage for the provided grain. + /// + /// The grain state type. + /// The grain context. + /// The grain storage for the provided grain. + IStorage GetStorage(IGrainContext grainContext); } diff --git a/src/Orleans.Core.Abstractions/Runtime/IGrainTimer.cs b/src/Orleans.Core.Abstractions/Runtime/IGrainTimer.cs index 7b752d4f1f..1b24200d3f 100644 --- a/src/Orleans.Core.Abstractions/Runtime/IGrainTimer.cs +++ b/src/Orleans.Core.Abstractions/Runtime/IGrainTimer.cs @@ -11,7 +11,8 @@ public interface IGrainTimer : IDisposable /// Changes the start time and the interval between method invocations for a timer, using values to measure time intervals. /// /// A representing the amount of time to delay before invoking the callback method specified when the was constructed. - /// Specify to prevent the timer from restarting. Specify to restart the timer immediately. + /// Specify to prevent the timer from restarting. + /// Specify to restart the timer immediately. /// /// /// The time interval between invocations of the callback method specified when the timer was constructed. diff --git a/src/Orleans.Core.Abstractions/Timers/GrainTimerCreationOptions.cs b/src/Orleans.Core.Abstractions/Timers/GrainTimerCreationOptions.cs new file mode 100644 index 0000000000..dd1f92a4f5 --- /dev/null +++ b/src/Orleans.Core.Abstractions/Timers/GrainTimerCreationOptions.cs @@ -0,0 +1,69 @@ +#nullable enable +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using Orleans.Concurrency; + +namespace Orleans.Runtime; + +/// +/// Options for creating grain timers. +/// +public readonly struct GrainTimerCreationOptions() +{ + /// + /// Initializes a new instance. + /// + /// + /// A representing the amount of time to delay before invoking the callback method specified when the was constructed. + /// Specify to prevent the timer from starting. + /// Specify to start the timer immediately. + /// + /// + /// The time interval between invocations of the callback method specified when the was constructed. + /// Specify to disable periodic signaling. + /// + [SetsRequiredMembers] + public GrainTimerCreationOptions(TimeSpan dueTime, TimeSpan period) : this() + { + DueTime = dueTime; + Period = period; + } + + /// + /// A representing the amount of time to delay before invoking the callback method specified when the was constructed. + /// Specify to prevent the timer from starting. + /// Specify to start the timer immediately. + /// + public required TimeSpan DueTime { get; init; } + + /// + /// The time interval between invocations of the callback method specified when the was constructed. + /// Specify to disable periodic signaling. + /// + public required TimeSpan Period { get; init; } + + /// + /// Gets a value indicating whether callbacks scheduled by this timer are allowed to interleave execution with other timers and grain calls. + /// Defaults to . + /// + /// + /// If this value is , the timer callback will be treated akin to a grain call. If the grain scheduling this timer is reentrant + /// (i.e., it has the attributed applied to its implementation class), the timer callback will be allowed + /// to interleave with other grain calls and timers regardless of the value of this property. + /// If this value is , the timer callback will be allowed to interleave with other timers and grain calls. + /// + public bool Interleave { get; init; } + + /// + /// Gets a value indicating whether callbacks scheduled by this timer should extend the lifetime of the grain activation. + /// Defaults to . + /// + /// + /// If this value is , timer callbacks will not extend a grain activation's lifetime. + /// If a grain is only processing this timer's callbacks and no other messages, the grain will be collected after its idle collection period expires. + /// If this value is , timer callback will extend a grain activation's lifetime. + /// If the timer period is shorter than the grain's idle collection period, the grain will not be collected due to idleness. + /// + public bool KeepAlive { get; init; } +} \ No newline at end of file diff --git a/src/Orleans.Core.Abstractions/Timers/ITimerRegistry.cs b/src/Orleans.Core.Abstractions/Timers/ITimerRegistry.cs index 85e44e6bd3..7789e3fe78 100644 --- a/src/Orleans.Core.Abstractions/Timers/ITimerRegistry.cs +++ b/src/Orleans.Core.Abstractions/Timers/ITimerRegistry.cs @@ -1,5 +1,6 @@ #nullable enable using System; +using System.Threading; using System.Threading.Tasks; using Orleans.Runtime; @@ -28,5 +29,10 @@ public interface ITimerRegistry /// /// An instance which represents the timer. /// + [Obsolete("Use 'RegisterGrainTimer(grainContext, callback, state, new() { DueTime = dueTime, Period = period, Interleave = true })' instead.")] IDisposable RegisterTimer(IGrainContext grainContext, Func callback, object? state, TimeSpan dueTime, TimeSpan period); -} \ No newline at end of file + + /// + /// The grain which the timer is associated with. + IGrainTimer RegisterGrainTimer(IGrainContext grainContext, Func callback, T state, GrainTimerCreationOptions options); +} diff --git a/src/Orleans.Core/Async/AsyncExecutorWithRetries.cs b/src/Orleans.Core/Async/AsyncExecutorWithRetries.cs index 1966cba1e1..305a1a1e0b 100644 --- a/src/Orleans.Core/Async/AsyncExecutorWithRetries.cs +++ b/src/Orleans.Core/Async/AsyncExecutorWithRetries.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.ExceptionServices; +using System.Threading; using System.Threading.Tasks; using Orleans.Runtime; @@ -211,7 +212,7 @@ public static class AsyncExecutorWithRetries { retry = false; - if (maxExecutionTime != Constants.INFINITE_TIMESPAN && maxExecutionTime != default) + if (maxExecutionTime != Timeout.InfiniteTimeSpan && maxExecutionTime != default) { DateTime now = DateTime.UtcNow; if (now - startExecutionTime > maxExecutionTime) diff --git a/src/Orleans.Core/Async/MultiTaskCompletionSource.cs b/src/Orleans.Core/Async/MultiTaskCompletionSource.cs index ef0b7b9640..4765da16ce 100644 --- a/src/Orleans.Core/Async/MultiTaskCompletionSource.cs +++ b/src/Orleans.Core/Async/MultiTaskCompletionSource.cs @@ -2,60 +2,53 @@ using System.Threading; using System.Threading.Tasks; -namespace Orleans +namespace Orleans; + +/// +/// An alternative to which completes only once a specified number of signals have been received. +/// +internal sealed class MultiTaskCompletionSource { + private readonly TaskCompletionSource _tcs; + private int _count; + /// - /// An alternative to which completes only once a specified number of signals have been received. + /// Initializes a new instance of the class. /// - internal class MultiTaskCompletionSource + /// + /// The number of signals which must occur before this completion source completes. + /// + /// + /// The count value is less than or equal to zero. + /// + public MultiTaskCompletionSource(int count) { - private readonly TaskCompletionSource tcs; - private int count; + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(count, 0); + _tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _count = count; + } - /// - /// Initializes a new instance of the class. - /// - /// - /// The number of signals which must occur before this completion source completes. - /// - /// - /// The count value is less than or equal to zero. - /// - public MultiTaskCompletionSource(int count) - { - if (count <= 0) - { - throw new ArgumentOutOfRangeException(nameof(count), "count has to be positive."); - } - tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - this.count = count; - } + /// + /// Gets the task which is completed when a sufficient number of signals are received. + /// + public Task Task => _tcs.Task; - /// - /// Gets the task which is completed when a sufficient number of signals are received. - /// - public Task Task + /// + /// Signals this instance. + /// + /// This method was called more times than the initially specified count argument allows. + public void SetOneResult() + { + var current = Interlocked.Decrement(ref _count); + if (current < 0) { - get { return tcs.Task; } + throw new InvalidOperationException( + "SetOneResult was called more times than initially specified by the count argument."); } - /// - /// Signals this instance. - /// - /// This method was called more times than the initially specified count argument allows. - public void SetOneResult() + if (current == 0) { - int current = Interlocked.Decrement(ref count); - if (current < 0) - { - throw new InvalidOperationException( - "SetOneResult was called more times than initially specified by the count argument."); - } - - if (current == 0) - { - tcs.SetResult(true); - } + _tcs.SetResult(); } } } diff --git a/src/Orleans.Core/Messaging/Message.cs b/src/Orleans.Core/Messaging/Message.cs index 6c82223d3b..ce6d13400d 100644 --- a/src/Orleans.Core/Messaging/Message.cs +++ b/src/Orleans.Core/Messaging/Message.cs @@ -92,6 +92,12 @@ public bool IsSystemMessage set => _headers.SetFlag(MessageFlags.SystemMessage, value); } + /// + /// Indicates whether the message does not mutate application state and therefore whether it can be interleaved with other read-only messages. + /// + /// + /// Defaults to . + /// public bool IsReadOnly { get => _headers.HasFlag(MessageFlags.ReadOnly); @@ -110,6 +116,30 @@ public bool IsUnordered set => _headers.SetFlag(MessageFlags.Unordered, value); } + /// + /// Whether the message is allowed to be sent to another activation of the target grain. + /// + /// + /// Defaults to . + /// + public bool IsLocalOnly + { + get => _headers.HasFlag(MessageFlags.IsLocalOnly); + set => _headers.SetFlag(MessageFlags.IsLocalOnly, value); + } + + /// + /// Whether the message is allowed to activate a grain and/or extend its lifetime. + /// + /// + /// Defaults to . + /// + public bool IsKeepAlive + { + get => !_headers.HasFlag(MessageFlags.SuppressKeepAlive); + set => _headers.SetFlag(MessageFlags.SuppressKeepAlive, !value); + } + public CorrelationId Id { get => _id; @@ -234,10 +264,8 @@ public GrainInterfaceType InterfaceType } } - public bool IsExpirableMessage(bool dropExpiredMessages) + public bool IsExpirableMessage() { - if (!dropExpiredMessages) return false; - GrainId id = TargetGrain; if (id.IsDefault) return false; @@ -339,6 +367,12 @@ internal enum MessageFlags : ushort HasCacheInvalidationHeader = 1 << 7, HasTimeToLive = 1 << 8, + // Message cannot be forwarded to another activation. + IsLocalOnly = 1 << 9, + + // Message must not trigger grain activation or extend an activation's lifetime. + SuppressKeepAlive = 1 << 10, + // The most significant bit is reserved, possibly for use to indicate more data follows. Reserved = 1 << 15, } @@ -385,7 +419,7 @@ public ResponseTypes ResponseType public void SetFlag(MessageFlags flag, bool value) => _fields = value switch { true => _fields | (uint)flag, - _ => _fields & ~(uint)flag, + false => _fields & ~(uint)flag, }; } } diff --git a/src/Orleans.Core/Networking/Connection.cs b/src/Orleans.Core/Networking/Connection.cs index d306e641d3..f4d9f24179 100644 --- a/src/Orleans.Core/Networking/Connection.cs +++ b/src/Orleans.Core/Networking/Connection.cs @@ -263,6 +263,7 @@ private async Task CloseAsync() public virtual void Send(Message message) { + Debug.Assert(!message.IsLocalOnly); if (!this.outgoingMessageWriter.TryWrite(message)) { this.RerouteMessage(message); diff --git a/src/Orleans.Core/Providers/GrainStorageHelpers.cs b/src/Orleans.Core/Providers/GrainStorageHelpers.cs index 63a11a85ac..d727f50486 100644 --- a/src/Orleans.Core/Providers/GrainStorageHelpers.cs +++ b/src/Orleans.Core/Providers/GrainStorageHelpers.cs @@ -1,49 +1,47 @@ +#nullable enable using System; using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using Orleans.Providers; -using Orleans.Runtime; -#nullable enable -namespace Orleans.Storage +namespace Orleans.Storage; + +/// +/// Utility functions for grain storage. +/// +public static class GrainStorageHelpers { /// - /// Utility functions for grain storage. + /// Gets the associated with the specified grain type, which must derive from . /// - public static class GrainStorageHelpers + /// The grain type, which must derive from . + /// The service provider. + /// + /// The associated with the specified grain type, which must derive from . + /// + public static IGrainStorage GetGrainStorage(Type grainType, IServiceProvider services) { - /// - /// Gets the associated with the specified grain type, which must derive from . - /// - /// The grain type, which must derive from . - /// The service provider. - /// - /// The associated with the specified grain type, which must derive from . - /// - public static IGrainStorage GetGrainStorage(Type grainType, IServiceProvider services) + if (grainType is null) throw new ArgumentNullException(nameof(grainType)); + var attrs = grainType.GetCustomAttributes(typeof(StorageProviderAttribute), true); + var attr = attrs.Length > 0 ? (StorageProviderAttribute)attrs[0] : null; + var storageProvider = attr != null + ? services.GetKeyedService(attr.ProviderName) + : services.GetService(); + if (storageProvider == null) { - if (grainType is null) throw new ArgumentNullException(nameof(grainType)); - var attrs = grainType.GetCustomAttributes(typeof(StorageProviderAttribute), true); - var attr = attrs.Length > 0 ? (StorageProviderAttribute)attrs[0] : null; - var storageProvider = attr != null - ? services.GetKeyedService(attr.ProviderName) - : services.GetService(); - if (storageProvider == null) - { - ThrowMissingProviderException(grainType, attr?.ProviderName); - } - - return storageProvider; + ThrowMissingProviderException(grainType, attr?.ProviderName); } - [DoesNotReturn] - private static void ThrowMissingProviderException(Type grainType, string? name) - { - var grainTypeName = grainType.FullName; - var errMsg = string.IsNullOrEmpty(name) - ? $"No default storage provider found loading grain type {grainTypeName}." - : $"No storage provider named \"{name}\" found loading grain type {grainTypeName}."; - throw new BadProviderConfigException(errMsg); - } + return storageProvider; + } + + [DoesNotReturn] + private static void ThrowMissingProviderException(Type grainType, string? name) + { + var grainTypeName = grainType.FullName; + var errMsg = string.IsNullOrEmpty(name) + ? $"No default storage provider found loading grain type {grainTypeName}." + : $"No storage provider named \"{name}\" found loading grain type {grainTypeName}."; + throw new BadProviderConfigException(errMsg); } } diff --git a/src/Orleans.Core/Runtime/AsyncEnumerableGrainExtension.cs b/src/Orleans.Core/Runtime/AsyncEnumerableGrainExtension.cs index b7ea7f53d1..3c25f8259f 100644 --- a/src/Orleans.Core/Runtime/AsyncEnumerableGrainExtension.cs +++ b/src/Orleans.Core/Runtime/AsyncEnumerableGrainExtension.cs @@ -33,12 +33,17 @@ public AsyncEnumerableGrainExtension(IGrainContext grainContext, IOptions(); - _timer = registry.RegisterTimer( + _timer = registry.RegisterGrainTimer( _grainContext, - static async state => await ((AsyncEnumerableGrainExtension)state).RemoveExpiredAsync(), + static async (state, cancellationToken) => await state.RemoveExpiredAsync(cancellationToken), this, - TimeSpan.FromSeconds(EnumeratorExpirationMilliseconds), - TimeSpan.FromSeconds(EnumeratorExpirationMilliseconds)); + new() + { + DueTime = TimeSpan.FromSeconds(EnumeratorExpirationMilliseconds), + Period = TimeSpan.FromSeconds(EnumeratorExpirationMilliseconds), + Interleave = true, + KeepAlive = false + }); } /// @@ -52,7 +57,7 @@ public ValueTask DisposeAsync(Guid requestId) return default; } - public async ValueTask RemoveExpiredAsync() + private async ValueTask RemoveExpiredAsync(CancellationToken cancellationToken) { List toRemove = default; foreach (var (requestId, state) in _enumerators) @@ -83,7 +88,7 @@ public async ValueTask RemoveExpiredAsync() if (tasks is { Count: > 0 }) { - await Task.WhenAll(tasks); + await Task.WhenAll(tasks).WaitAsync(cancellationToken); } } diff --git a/src/Orleans.Core/Runtime/ClientGrainContext.cs b/src/Orleans.Core/Runtime/ClientGrainContext.cs index 18245ad21b..e228c7d212 100644 --- a/src/Orleans.Core/Runtime/ClientGrainContext.cs +++ b/src/Orleans.Core/Runtime/ClientGrainContext.cs @@ -183,8 +183,8 @@ public void ReceiveMessage(object message) throw new NotImplementedException(); } - public void Activate(Dictionary requestContext, CancellationToken? cancellationToken = null) { } - public void Deactivate(DeactivationReason deactivationReason, CancellationToken? cancellationToken = null) { } + public void Activate(Dictionary requestContext, CancellationToken cancellationToken) { } + public void Deactivate(DeactivationReason deactivationReason, CancellationToken cancellationToken) { } public void Rehydrate(IRehydrationContext context) { @@ -192,7 +192,7 @@ public void Rehydrate(IRehydrationContext context) (context as IDisposable)?.Dispose(); } - public void Migrate(Dictionary requestContext, CancellationToken? cancellationToken = null) + public void Migrate(Dictionary requestContext, CancellationToken cancellationToken) { // Migration is not supported. Do nothing: the contract is that this method attempts migration, but does not guarantee it will occur. } diff --git a/src/Orleans.Core/Runtime/Constants.cs b/src/Orleans.Core/Runtime/Constants.cs index 03e69fde38..fbba2f9bc6 100644 --- a/src/Orleans.Core/Runtime/Constants.cs +++ b/src/Orleans.Core/Runtime/Constants.cs @@ -1,30 +1,11 @@ using System; +using System.Collections.Frozen; using System.Collections.Generic; namespace Orleans.Runtime { internal static class Constants { - // This needs to be first, as GrainId static initializers reference it. Otherwise, GrainId actually see a uninitialized (ie Zero) value for that "constant"! - public static readonly TimeSpan INFINITE_TIMESPAN = TimeSpan.FromMilliseconds(-1); - - // We assume that clock skew between silos and between clients and silos is always less than 1 second - public static readonly TimeSpan MAXIMUM_CLOCK_SKEW = TimeSpan.FromSeconds(1); - - public const string DATA_CONNECTION_STRING_NAME = "DataConnectionString"; - public const string ADO_INVARIANT_NAME = "AdoInvariant"; - public const string DATA_CONNECTION_FOR_REMINDERS_STRING_NAME = "DataConnectionStringForReminders"; - public const string ADO_INVARIANT_FOR_REMINDERS_NAME = "AdoInvariantForReminders"; - - public const string ORLEANS_CLUSTERING_AZURESTORAGE = "Orleans.Clustering.AzureStorage"; - public const string ORLEANS_REMINDERS_AZURESTORAGE = "Orleans.Reminders.AzureStorage"; - - public const string ORLEANS_CLUSTERING_ADONET = "Orleans.Clustering.AdoNet"; - public const string ORLEANS_REMINDERS_ADONET = "Orleans.Reminders.AdoNet"; - - public const string INVARIANT_NAME_SQL_SERVER = "System.Data.SqlClient"; - - public const string ORLEANS_CLUSTERING_ZOOKEEPER = "Orleans.Clustering.ZooKeeper"; public const string TroubleshootingHelpLink = "https://aka.ms/orleans-troubleshooting"; public static readonly GrainType DirectoryServiceType = SystemTargetGrainId.CreateGrainType("dir.mem"); @@ -49,13 +30,9 @@ internal static class Constants GrainType.Create(GrainTypePrefix.SystemPrefix + "silo"), IdSpan.Create("01111111-1111-1111-1111-111111111111")); - public const int LARGE_OBJECT_HEAP_THRESHOLD = 85000; - - public const int DEFAULT_LOGGER_BULK_MESSAGE_LIMIT = 5; - public static readonly TimeSpan DEFAULT_CLIENT_DROP_TIMEOUT = TimeSpan.FromMinutes(1); - private static readonly Dictionary singletonSystemTargetNames = new Dictionary + private static readonly FrozenDictionary SingletonSystemTargetNames = new Dictionary { {DirectoryServiceType, "DirectoryService"}, {DirectoryCacheValidatorType, "DirectoryCacheValidator"}, @@ -73,16 +50,10 @@ internal static class Constants {StreamPullingAgentManagerType, "PullingAgentsManagerSystemTarget"}, {StreamPullingAgentType, "PullingAgentSystemTarget"}, {ManifestProviderType, "ManifestProvider"}, - }; - - public static ushort DefaultInterfaceVersion = 1; + }.ToFrozenDictionary(); - public static string SystemTargetName(GrainType id) => singletonSystemTargetNames.TryGetValue(id, out var name) ? name : id.ToString(); - - public static bool IsSingletonSystemTarget(GrainType id) - { - return singletonSystemTargetNames.ContainsKey(id); - } + public static string SystemTargetName(GrainType id) => SingletonSystemTargetNames.TryGetValue(id, out var name) ? name : id.ToString(); + public static bool IsSingletonSystemTarget(GrainType id) => SingletonSystemTargetNames.ContainsKey(id); } } diff --git a/src/Orleans.Core/Runtime/InvokableObjectManager.cs b/src/Orleans.Core/Runtime/InvokableObjectManager.cs index 2946f949e4..a729e3f6ce 100644 --- a/src/Orleans.Core/Runtime/InvokableObjectManager.cs +++ b/src/Orleans.Core/Runtime/InvokableObjectManager.cs @@ -340,8 +340,8 @@ private void ReportException(Message message, Exception exception) } } - public void Activate(Dictionary requestContext, CancellationToken? cancellationToken = null) { } - public void Deactivate(DeactivationReason deactivationReason, CancellationToken? cancellationToken = null) { } + public void Activate(Dictionary requestContext, CancellationToken cancellationToken) { } + public void Deactivate(DeactivationReason deactivationReason, CancellationToken cancellationToken) { } public void Rehydrate(IRehydrationContext context) { @@ -349,7 +349,7 @@ public void Rehydrate(IRehydrationContext context) (context as IDisposable)?.Dispose(); } - public void Migrate(Dictionary requestContext, CancellationToken? cancellationToken = null) + public void Migrate(Dictionary requestContext, CancellationToken cancellationToken) { // Migration is not supported. Do nothing: the contract is that this method attempts migration, but does not guarantee it will occur. } diff --git a/src/Orleans.Core/Runtime/OutsideRuntimeClient.cs b/src/Orleans.Core/Runtime/OutsideRuntimeClient.cs index b2fedf7b36..9d1a3cfd20 100644 --- a/src/Orleans.Core/Runtime/OutsideRuntimeClient.cs +++ b/src/Orleans.Core/Runtime/OutsideRuntimeClient.cs @@ -264,7 +264,7 @@ public void SendRequest(GrainReference target, IInvokable request, IResponseComp message.TargetSilo = systemTargetGrainId.GetSiloAddress(); } - if (message.IsExpirableMessage(this.clientMessagingOptions.DropExpiredMessages)) + if (this.clientMessagingOptions.DropExpiredMessages && message.IsExpirableMessage()) { // don't set expiration for system target messages. var ttl = request.GetDefaultResponseTimeout() ?? this.clientMessagingOptions.ResponseTimeout; diff --git a/src/Orleans.Runtime/Catalog/ActivationCollector.cs b/src/Orleans.Runtime/Catalog/ActivationCollector.cs index 72953b5d78..ea8f7e8c55 100644 --- a/src/Orleans.Runtime/Catalog/ActivationCollector.cs +++ b/src/Orleans.Runtime/Catalog/ActivationCollector.cs @@ -8,13 +8,14 @@ using Microsoft.Extensions.Options; using Orleans.Configuration; using Orleans.Internal; +using Orleans.Runtime.Internal; namespace Orleans.Runtime { /// /// Identifies activations that have been idle long enough to be deactivated. /// - internal class ActivationCollector : IActivationWorkingSetObserver, IHealthCheckParticipant, ILifecycleParticipant + internal class ActivationCollector : IActivationWorkingSetObserver, ILifecycleParticipant { internal Action Debug_OnDecideToCollectActivation; private readonly TimeSpan quantum; @@ -23,7 +24,7 @@ internal class ActivationCollector : IActivationWorkingSetObserver, IHealthCheck private DateTime nextTicket; private static readonly List nothing = new(0); private readonly ILogger logger; - private readonly IAsyncTimer _collectionTimer; + private readonly PeriodicTimer _collectionTimer; private Task _collectionLoopTask; private int collectionNumber; private int _activationCount; @@ -45,7 +46,7 @@ internal class ActivationCollector : IActivationWorkingSetObserver, IHealthCheck shortestAgeLimit = new(options.Value.ClassSpecificCollectionAge.Values.Aggregate(options.Value.CollectionAge.Ticks, (a, v) => Math.Min(a, v.Ticks))); nextTicket = MakeTicketFromDateTime(DateTime.UtcNow); this.logger = logger; - _collectionTimer = timerFactory.Create(quantum, "ActivationCollector"); + _collectionTimer = new PeriodicTimer(quantum); } // Return the number of activations that were used (touched) in the last recencyPeriod. @@ -421,13 +422,14 @@ void IActivationWorkingSetObserver.OnDeactivated(IActivationWorkingSetMember mem private Task Start(CancellationToken cancellationToken) { + using var _ = new ExecutionContextSuppressor(); _collectionLoopTask = RunActivationCollectionLoop(); return Task.CompletedTask; } private async Task Stop(CancellationToken cancellationToken) { - _collectionTimer?.Dispose(); + _collectionTimer.Dispose(); if (_collectionLoopTask is Task task) { @@ -446,8 +448,8 @@ void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) private async Task RunActivationCollectionLoop() { - while (await _collectionTimer.NextTick()) - + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); + while (await _collectionTimer.WaitForNextTickAsync()) { try { @@ -455,7 +457,7 @@ private async Task RunActivationCollectionLoop() } catch (Exception exception) { - this.logger.LogError(exception, "Exception while collecting activations"); + this.logger.LogError(exception, "Error while collecting activations."); } } } @@ -525,18 +527,6 @@ private async Task DeactivateActivationsFromCollector(List - public bool CheckHealth(DateTime lastCheckTime, out string reason) - { - if (_collectionTimer is IAsyncTimer timer) - { - return timer.CheckHealth(lastCheckTime, out reason); - } - - reason = default; - return true; - } - private class Bucket { public ConcurrentDictionary Items { get; } = new(ReferenceEqualsComparer.Default); diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 575abffa60..4947637a71 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1,6 +1,8 @@ +#nullable enable using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -25,7 +27,7 @@ namespace Orleans.Runtime; /// MUST lock this object for any concurrent access /// Consider: compartmentalize by usage, e.g., using separate interfaces for data for catalog, etc. /// -internal sealed class ActivationData : IGrainContext, ICollectibleGrainContext, IGrainExtensionBinder, IActivationWorkingSetMember, IGrainTimerRegistry, IGrainManagementExtension, ICallChainReentrantGrainContext, IAsyncDisposable +internal sealed class ActivationData : IGrainContext, ICollectibleGrainContext, IGrainExtensionBinder, IActivationWorkingSetMember, IGrainTimerRegistry, IGrainManagementExtension, ICallChainReentrantGrainContext, IAsyncDisposable, IDisposable { private const string GrainAddressMigrationContextKey = "sys.addr"; private readonly GrainTypeSharedContext _shared; @@ -34,18 +36,18 @@ internal sealed class ActivationData : IGrainContext, ICollectibleGrainContext, private readonly List<(Message Message, CoarseStopwatch QueuedTime)> _waitingRequests = new(); private readonly Dictionary _runningRequests = new(); private readonly SingleWaiterAutoResetEvent _workSignal = new() { RunContinuationsAsynchronously = true }; - private GrainLifecycle _lifecycle; - private List _pendingOperations; - private Message _blockingRequest; + private GrainLifecycle? _lifecycle; + private List? _pendingOperations; + private Message? _blockingRequest; private bool _isInWorkingSet; private CoarseStopwatch _busyDuration; private CoarseStopwatch _idleDuration; - private GrainReference _selfReference; + private GrainReference? _selfReference; // Values which are needed less frequently and do not warrant living directly on activation for object size reasons. // The values in this field are typically used to represent termination state of an activation or features which are not // used by all grains, such as grain timers. - private ActivationDataExtra _extras; + private ActivationDataExtra? _extras; // The task representing this activation's message loop. // This field is assigned and never read and exists only for debugging purposes (eg, in memory dumps, to associate a loop task with an activation). @@ -69,7 +71,7 @@ internal sealed class ActivationData : IGrainContext, ICollectibleGrainContext, } public IGrainRuntime GrainRuntime => _shared.Runtime; - public object GrainInstance { get; private set; } + public object? GrainInstance { get; private set; } public GrainAddress Address { get; } public GrainReference GrainReference => _selfReference ??= _shared.GrainReferenceActivator.CreateReference(GrainId, default); public ActivationState State { get; private set; } @@ -107,7 +109,7 @@ internal GrainLifecycle Lifecycle public IWorkItemScheduler Scheduler => _workItemGroup; public Task Deactivated => GetDeactivationCompletionSource().Task; - public SiloAddress ForwardingAddress + public SiloAddress? ForwardingAddress { get => _extras?.ForwardingAddress; set @@ -124,7 +126,7 @@ public SiloAddress ForwardingAddress /// Gets the previous directory registration for this grain, if known. /// This is used to update the grain directory to point to the new registration during activation. /// - public GrainAddress PreviousRegistration + public GrainAddress? PreviousRegistration { get => _extras?.PreviousRegistration; set @@ -137,7 +139,7 @@ public GrainAddress PreviousRegistration } } - private Exception DeactivationException => _extras?.DeactivationReason.Exception; + private Exception? DeactivationException => _extras?.DeactivationReason.Exception; private DeactivationReason DeactivationReason { @@ -152,7 +154,7 @@ private DeactivationReason DeactivationReason } } - private HashSet Timers + private HashSet? Timers { get => _extras?.Timers; set @@ -204,7 +206,7 @@ private bool IsStuckProcessingMessage } } - private DehydrationContextHolder DehydrationContext + private DehydrationContextHolder? DehydrationContext { get => _extras?.DehydrationContext; set @@ -219,9 +221,9 @@ private DehydrationContextHolder DehydrationContext public TimeSpan CollectionAgeLimit => _shared.CollectionAgeLimit; - public TTarget GetTarget() where TTarget : class => (TTarget)GrainInstance; + public TTarget? GetTarget() where TTarget : class => (TTarget?)GrainInstance; - TComponent ITargetHolder.GetComponent() + TComponent? ITargetHolder.GetComponent() where TComponent : class { var result = GetComponent(); if (result is null && typeof(IGrainExtension).IsAssignableFrom(typeof(TComponent))) @@ -239,9 +241,9 @@ TComponent ITargetHolder.GetComponent() return result; } - public TComponent GetComponent() where TComponent : class + public TComponent? GetComponent() where TComponent : class { - TComponent result; + TComponent? result; if (GrainInstance is TComponent grainResult) { result = grainResult; @@ -267,7 +269,7 @@ TComponent ITargetHolder.GetComponent() return result; } - public void SetComponent(TComponent instance) where TComponent : class + public void SetComponent(TComponent? instance) where TComponent : class { if (GrainInstance is TComponent) { @@ -292,7 +294,7 @@ TComponent ITargetHolder.GetComponent() } } - internal void SetGrainInstance(object grainInstance) + internal void SetGrainInstance(object? grainInstance) { switch (GrainInstance, grainInstance) { @@ -324,7 +326,7 @@ public void SetState(ActivationState state) /// Returns LimitExceededException if overloaded, otherwise nullc> /// /// Returns LimitExceededException if overloaded, otherwise nullc> - public LimitExceededException CheckOverloaded() + public LimitExceededException? CheckOverloaded() { string limitName = nameof(SiloMessagingOptions.MaxEnqueuedRequestsHardLimit); int maxRequestsHardLimit = _shared.MessagingOptions.MaxEnqueuedRequestsHardLimit; @@ -378,9 +380,20 @@ internal List DequeueAllWaitingRequests() { lock (this) { - var tmp = _waitingRequests.Select(m => m.Item1).ToList(); + var result = new List(_waitingRequests.Count); + foreach (var (message, _) in _waitingRequests) + { + // Local-only messages are not allowed to escape the activation. + if (message.IsLocalOnly) + { + continue; + } + + result.Add(message); + } + _waitingRequests.Clear(); - return tmp; + return result; } } @@ -408,7 +421,7 @@ public void DelayDeactivation(TimeSpan timespan) } else { - KeepAliveUntil = DateTime.UtcNow + timespan; + KeepAliveUntil = GrainRuntime.TimeProvider.GetUtcNow().UtcDateTime + timespan; } } @@ -428,18 +441,16 @@ private void ScheduleOperation(object operation) _workSignal.Signal(); } - public void Migrate(Dictionary requestContext, CancellationToken? cancellationToken = default) + public void Migrate(Dictionary? requestContext, CancellationToken cancellationToken = default) { - if (!cancellationToken.HasValue) - { - cancellationToken = new CancellationTokenSource(_shared.InternalRuntime.CollectionOptions.Value.DeactivationTimeout).Token; - } + var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(_shared.InternalRuntime.CollectionOptions.Value.DeactivationTimeout); // We use a named work item since it is cheaper than allocating a Task and has the benefit of being named. - _workItemGroup.QueueWorkItem(new MigrateWorkItem(this, requestContext, cancellationToken.Value)); + _workItemGroup.QueueWorkItem(new MigrateWorkItem(this, requestContext, cts)); } - private async Task StartMigratingAsync(Dictionary requestContext, CancellationToken cancellationToken) + private async Task StartMigratingAsync(Dictionary? requestContext, CancellationTokenSource cts) { lock (this) { @@ -455,7 +466,7 @@ private async Task StartMigratingAsync(Dictionary requestContext { // Run placement to select a new host. If a new (different) host is not selected, do not migrate. var placementService = _shared.Runtime.ServiceProvider.GetRequiredService(); - newLocation = await placementService.PlaceGrainAsync(GrainId, requestContext, PlacementStrategy); + newLocation = await placementService.PlaceGrainAsync(GrainId, requestContext, PlacementStrategy).WaitAsync(cts.Token); if (newLocation == Address.SiloAddress || newLocation is null) { // No more appropriate silo was selected for this grain. The migration attempt will be aborted. @@ -503,7 +514,7 @@ private async Task StartMigratingAsync(Dictionary requestContext } // Start deactivation to prevent any other. - ScheduleOperation(new Command.Deactivate(cancellationToken)); + ScheduleOperation(new Command.Deactivate(cts)); } catch (Exception exception) { @@ -512,15 +523,13 @@ private async Task StartMigratingAsync(Dictionary requestContext } } - public void Deactivate(DeactivationReason reason, CancellationToken? token = default) + public void Deactivate(DeactivationReason reason, CancellationToken cancellationToken) { - if (!token.HasValue) - { - token = new CancellationTokenSource(_shared.InternalRuntime.CollectionOptions.Value.DeactivationTimeout).Token; - } + var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(_shared.InternalRuntime.CollectionOptions.Value.DeactivationTimeout); StartDeactivating(reason); - ScheduleOperation(new Command.Deactivate(token.Value)); + ScheduleOperation(new Command.Deactivate(cts)); } private void DeactivateStuckActivation() @@ -530,7 +539,7 @@ private void DeactivateStuckActivation() var reason = new DeactivationReason(DeactivationReasonCode.ActivationUnresponsive, msg); // Mark the grain as deactivating so that messages are forwarded instead of being invoked - Deactivate(reason, token: default); + Deactivate(reason, cancellationToken: default); // Try to remove this activation from the catalog and directory // This leaves this activation dangling, stuck processing the current request until it eventually completes @@ -559,14 +568,10 @@ void IGrainTimerRegistry.OnTimerDisposed(IGrainTimer orleansTimerInsideGrain) } Timers.Remove(orleansTimerInsideGrain); - if (Timers.Count == 0) - { - Timers = null; - } } } - private void StopAllTimers() + private void DisposeTimers() { lock (this) { @@ -575,41 +580,15 @@ private void StopAllTimers() return; } - foreach (var timer in Timers) - { - timer.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); - } - } - } - - private Task WaitForAllTimersToFinish(CancellationToken cancellationToken) - { - lock (this) - { - if (Timers is null) - { - return Task.CompletedTask; - } + // Need to set Timers to null since OnTimerDisposed mutates the timers set if it is not null. + var timers = Timers; + Timers = null; - var tasks = new List(); - var timerCopy = Timers.ToList(); // need to copy since OnTimerDisposed will change the timers set. - foreach (var timer in timerCopy) + // Dispose all timers. + foreach (var timer in timers) { - if (timer is IAsyncDisposable asyncDisposable) - { - var task = asyncDisposable.DisposeAsync(); - if (!task.IsCompletedSuccessfully) - { - tasks.Add(task.AsTask()); - } - } - else - { - timer.Dispose(); - } + timer.Dispose(); } - - return Task.WhenAll(tasks).WithCancellation(cancellationToken); } } @@ -618,7 +597,7 @@ public void AnalyzeWorkload(DateTime now, IMessageCenter messageCenter, MessageF var slowRunningRequestDuration = options.RequestProcessingWarningTime; var longQueueTimeDuration = options.RequestQueueDelayWarningTime; - List diagnostics = null; + List? diagnostics = null; lock (this) { if (State != ActivationState.Valid) @@ -697,7 +676,7 @@ public void AnalyzeWorkload(DateTime now, IMessageCenter messageCenter, MessageF } } - void GetStatusList(ref List diagnostics) + void GetStatusList([NotNull] ref List? diagnostics) { if (diagnostics is not null) return; @@ -731,12 +710,16 @@ private string GetActivationInfoString() return grainTypeName is null ? $"#Placement={placement}" : $"#GrainType={grainTypeName} Placement={placement}"; } + public void Dispose() => DisposeAsync().AsTask().Wait(); + public async ValueTask DisposeAsync() { _extras ??= new(); if (_extras.IsDisposing) return; _extras.IsDisposing = true; + DisposeTimers(); + try { var activator = GetComponent(); @@ -768,7 +751,7 @@ public async ValueTask DisposeAsync() } } - bool IEquatable.Equals(IGrainContext other) => ReferenceEquals(this, other); + bool IEquatable.Equals(IGrainContext? other) => ReferenceEquals(this, other); public (TExtension, TExtensionInterface) GetOrSetExtension(Func newExtensionFunc) where TExtension : class, TExtensionInterface @@ -843,7 +826,7 @@ private async Task RunMessageLoop() { if (!IsCurrentlyExecuting) { - List operations = null; + List? operations = null; lock (this) { if (_pendingOperations is { Count: > 0 }) @@ -875,7 +858,7 @@ void ProcessPendingRequests() do { - Message message = null; + Message? message = null; lock (this) { if (_waitingRequests.Count <= i) @@ -883,13 +866,16 @@ void ProcessPendingRequests() break; } - if (State != ActivationState.Valid) + message = _waitingRequests[i].Message; + + // If the activation is not valid, reject all pending messages except for local-only messages. + // Local-only messages are used for internal system operations and should not be rejected. + if (State != ActivationState.Valid && !message.IsLocalOnly) { ProcessRequestsToInvalidActivation(); break; } - message = _waitingRequests[i].Message; try { if (!MayInvokeRequest(message)) @@ -935,14 +921,18 @@ void ProcessPendingRequests() DeactivationReasonCode.IncompatibleRequest, $"Received incompatible request for interface {message.InterfaceType} version {message.InterfaceVersion}. This activation supports interface version {currentVersion}."); - Deactivate(reason, token: default); + Deactivate(reason, cancellationToken: default); return; } } } catch (Exception exception) { - _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Transient, exception); + if (!message.IsLocalOnly) + { + _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Transient, exception); + } + _waitingRequests.RemoveAt(i); continue; } @@ -950,7 +940,7 @@ void ProcessPendingRequests() // Process this message, removing it from the queue. _waitingRequests.RemoveAt(i); - Debug.Assert(State == ActivationState.Valid); + Debug.Assert(State == ActivationState.Valid || message.IsLocalOnly); RecordRunning(message, message.IsAlwaysInterleave); } @@ -984,7 +974,7 @@ void ProcessRequestsToInvalidActivation() if (State is ActivationState.Deactivating) { // Determine whether to declare this activation as stuck - var deactivatingTime = DateTime.UtcNow - DeactivationStartTime.Value; + var deactivatingTime = GrainRuntime.TimeProvider.GetUtcNow().UtcDateTime - DeactivationStartTime!.Value; if (deactivatingTime > _shared.MaxRequestProcessingTime && !IsStuckDeactivating) { IsStuckDeactivating = true; @@ -1074,13 +1064,28 @@ async Task ProcessOperationsAsync(List operations) RehydrateInternal(command.Context); break; case Command.Activate command: - await ActivateAsync(command.RequestContext, command.CancellationToken); + try + { + await ActivateAsync(command.RequestContext, command.Cts.Token); + } + finally + { + command.Cts.Dispose(); + } break; case Command.Deactivate command: - await FinishDeactivating(command.CancellationToken); + try + { + await FinishDeactivating(command.Cts.Token); + } + finally + { + command.Cts.Dispose(); + } + break; case Command.Delay command: - await Task.Delay(command.Duration); + await Task.Delay(command.Duration, GrainRuntime.TimeProvider); break; case Command.UnregisterFromCatalog: UnregisterMessageTarget(); @@ -1113,7 +1118,7 @@ private void RehydrateInternal(IRehydrationContext context) throw new InvalidOperationException($"Attempted to rehydrate a grain in the {State} state"); } - if (context.TryGetValue(GrainAddressMigrationContextKey, out GrainAddress previousRegistration) && previousRegistration is not null) + if (context.TryGetValue(GrainAddressMigrationContextKey, out GrainAddress? previousRegistration) && previousRegistration is not null) { // Propagate the previous registration, so that the new activation can atomically replace it with its new address. PreviousRegistration = previousRegistration; @@ -1230,23 +1235,24 @@ static async ValueTask OnCompleteAsync(ActivationData activation, Message messag /// /// Invoked when an activation has finished a transaction and may be ready for additional transactions /// - /// The message that has just completed processing. - /// This will be null for the case of completion of Activate/Deactivate calls. + /// The message that has just completed processing. private void OnCompletedRequest(Message message) { lock (this) { _runningRequests.Remove(message); - if (_runningRequests.Count == 0) + // If the message is meant to keep the activation active, reset the idle timer and ensure the activation + // is in the activation working set. + if (message.IsKeepAlive) { _idleDuration = CoarseStopwatch.StartNew(); - } - if (!_isInWorkingSet) - { - _isInWorkingSet = true; - _shared.InternalRuntime.ActivationWorkingSet.OnActive(this); + if (!_isInWorkingSet) + { + _isInWorkingSet = true; + _shared.InternalRuntime.ActivationWorkingSet.OnActive(this); + } } // The below logic only works for non-reentrant activations @@ -1305,7 +1311,7 @@ private void ReceiveResponse(Message message) private void ReceiveRequest(Message message) { var overloadException = CheckOverloaded(); - if (overloadException != null) + if (overloadException != null && !message.IsLocalOnly) { MessagingProcessingInstruments.OnDispatcherMessageProcessedError(message); _shared.InternalRuntime.MessageCenter.RejectMessage(message, Message.RejectionTypes.Overloaded, overloadException, "Target activation is overloaded " + this); @@ -1381,17 +1387,15 @@ public void Rehydrate(IRehydrationContext context) ScheduleOperation(new Command.Rehydrate(context)); } - public void Activate(Dictionary requestContext, CancellationToken? cancellationToken) + public void Activate(Dictionary? requestContext, CancellationToken cancellationToken) { - if (!cancellationToken.HasValue) - { - cancellationToken = new CancellationTokenSource(_shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout).Token; - } + var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(_shared.InternalRuntime.CollectionOptions.Value.ActivationTimeout); - ScheduleOperation(new Command.Activate(requestContext, cancellationToken.Value)); + ScheduleOperation(new Command.Activate(requestContext, cts)); } - private async Task ActivateAsync(Dictionary requestContextData, CancellationToken cancellationToken) + private async Task ActivateAsync(Dictionary? requestContextData, CancellationToken cancellationToken) { // A chain of promises that will have to complete in order to complete the activation // Register with the grain directory, register with the store if necessary and call the Activate method on the new activation. @@ -1431,7 +1435,7 @@ private async Task ActivateAsync(Dictionary requestContextData, _workSignal.Signal(); } - async Task CallActivateAsync(Dictionary requestContextData, CancellationToken cancellationToken) + async Task CallActivateAsync(Dictionary? requestContextData, CancellationToken cancellationToken) { if (_shared.Logger.IsEnabled(LogLevel.Debug)) { @@ -1506,7 +1510,7 @@ async Task CallActivateAsync(Dictionary requestContextData ScheduleOperation(new Command.Delay(TimeSpan.FromSeconds(5))); } - ScheduleOperation(new Command.UnregisterFromCatalog()); + ScheduleOperation(Command.UnregisterFromCatalog.Instance); lock (this) { @@ -1531,7 +1535,7 @@ private async ValueTask RegisterActivationInGrainDirectoryAndValidate() } else { - Exception registrationException; + Exception? registrationException; var previousRegistration = PreviousRegistration; try { @@ -1649,12 +1653,8 @@ public bool StartDeactivating(DeactivationReason reason) DeactivationReason = reason; } - DeactivationStartTime = DateTime.UtcNow; + DeactivationStartTime = GrainRuntime.TimeProvider.GetUtcNow().UtcDateTime; SetState(ActivationState.Deactivating); - if (!IsCurrentlyExecuting) - { - StopAllTimers(); - } _shared.InternalRuntime.ActivationWorkingSet.OnDeactivating(this); } @@ -1676,10 +1676,10 @@ private async Task FinishDeactivating(CancellationToken cancellationToken) _shared.Logger.LogTrace("FinishDeactivating activation {Activation}", this.ToDetailedString()); } - StopAllTimers(); + // Stop timers from firing. + DisposeTimers(); - // Wait timers and call OnDeactivateAsync(reason, cancellationToken) - await WaitForAllTimersToFinish(cancellationToken); + // Call OnDeactivateAsync(reason, cancellationToken) await CallGrainDeactivate(cancellationToken); if (DehydrationContext is { } context @@ -1841,13 +1841,13 @@ private TaskCompletionSource GetDeactivationCompletionSource() ValueTask IGrainManagementExtension.DeactivateOnIdle() { - Deactivate(new(DeactivationReasonCode.ApplicationRequested, $"{nameof(IGrainManagementExtension.DeactivateOnIdle)} was called.")); + Deactivate(new(DeactivationReasonCode.ApplicationRequested, $"{nameof(IGrainManagementExtension.DeactivateOnIdle)} was called."), CancellationToken.None); return default; } ValueTask IGrainManagementExtension.MigrateOnIdle() { - Migrate(RequestContext.CallContextData?.Value.Values); + Migrate(RequestContext.CallContextData?.Value.Values, CancellationToken.None); return default; } @@ -1907,22 +1907,22 @@ private class ActivationDataExtra : Dictionary private const int IsDisposingFlag = 1 << 2; private byte _flags; - public HashSet Timers { get => GetValueOrDefault>(nameof(Timers)); set => SetOrRemoveValue(nameof(Timers), value); } + public HashSet? Timers { get => GetValueOrDefault>(nameof(Timers)); set => SetOrRemoveValue(nameof(Timers), value); } /// /// During rehydration, this may contain the address for the previous (recently dehydrated) activation of this grain. /// - public GrainAddress PreviousRegistration { get => GetValueOrDefault(nameof(PreviousRegistration)); set => SetOrRemoveValue(nameof(PreviousRegistration), value); } + public GrainAddress? PreviousRegistration { get => GetValueOrDefault(nameof(PreviousRegistration)); set => SetOrRemoveValue(nameof(PreviousRegistration), value); } /// /// If State == Invalid, this may contain a forwarding address for incoming messages /// - public SiloAddress ForwardingAddress { get => GetValueOrDefault(nameof(ForwardingAddress)); set => SetOrRemoveValue(nameof(ForwardingAddress), value); } + public SiloAddress? ForwardingAddress { get => GetValueOrDefault(nameof(ForwardingAddress)); set => SetOrRemoveValue(nameof(ForwardingAddress), value); } /// /// A which completes when a grain has deactivated. /// - public TaskCompletionSource DeactivationTask { get => GetDeactivationInfoOrDefault()?.DeactivationTask; set => EnsureDeactivationInfo().DeactivationTask = value; } + public TaskCompletionSource? DeactivationTask { get => GetDeactivationInfoOrDefault()?.DeactivationTask; set => EnsureDeactivationInfo().DeactivationTask = value; } public DateTime? DeactivationStartTime { get => GetDeactivationInfoOrDefault()?.DeactivationStartTime; set => EnsureDeactivationInfo().DeactivationStartTime = value; } @@ -1931,16 +1931,13 @@ private class ActivationDataExtra : Dictionary /// /// When migrating to another location, this contains the information to preserve across activations. /// - public DehydrationContextHolder DehydrationContext { get => GetValueOrDefault(nameof(DehydrationContext)); set => SetOrRemoveValue(nameof(DehydrationContext), value); } + public DehydrationContextHolder? DehydrationContext { get => GetValueOrDefault(nameof(DehydrationContext)); set => SetOrRemoveValue(nameof(DehydrationContext), value); } - private DeactivationInfo GetDeactivationInfoOrDefault() => GetValueOrDefault(nameof(DeactivationInfo)); + private DeactivationInfo? GetDeactivationInfoOrDefault() => GetValueOrDefault(nameof(DeactivationInfo)); private DeactivationInfo EnsureDeactivationInfo() { - if (!TryGetValue(nameof(DeactivationInfo), out var info)) - { - info = base[nameof(DeactivationInfo)] = new DeactivationInfo(); - } - + ref var info = ref CollectionsMarshal.GetValueRefOrAddDefault(this, nameof(DeactivationInfo), out _); + info ??= new DeactivationInfo(); return (DeactivationInfo)info; } @@ -1961,13 +1958,13 @@ private void SetFlag(int flag, bool value) } private bool GetFlag(int flag) => (_flags & flag) != 0; - private T GetValueOrDefault(object key) + private T? GetValueOrDefault(object key) { TryGetValue(key, out var result); - return (T)result; + return (T?)result; } - private void SetOrRemoveValue(object key, object value) + private void SetOrRemoveValue(object key, object? value) { if (value is null) { @@ -1983,7 +1980,7 @@ private sealed class DeactivationInfo { public DateTime? DeactivationStartTime; public DeactivationReason DeactivationReason; - public TaskCompletionSource DeactivationTask; + public TaskCompletionSource? DeactivationTask; } } @@ -1991,29 +1988,30 @@ private class Command { protected Command() { } - public class Deactivate(CancellationToken cancellation) : Command + public sealed class Deactivate(CancellationTokenSource cts) : Command { - public CancellationToken CancellationToken { get; } = cancellation; + public CancellationTokenSource Cts { get; } = cts; } - public class Activate(Dictionary requestContext, CancellationToken cancellationToken) : Command + public sealed class Activate(Dictionary? requestContext, CancellationTokenSource cts) : Command { - public Dictionary RequestContext { get; } = requestContext; - public CancellationToken CancellationToken { get; } = cancellationToken; + public Dictionary? RequestContext { get; } = requestContext; + public CancellationTokenSource Cts { get; } = cts; } - public class Rehydrate(IRehydrationContext context) : Command + public sealed class Rehydrate(IRehydrationContext context) : Command { public readonly IRehydrationContext Context = context; } - public class Delay(TimeSpan duration) : Command + public sealed class Delay(TimeSpan duration) : Command { public TimeSpan Duration { get; } = duration; } - public class UnregisterFromCatalog : Command + public sealed class UnregisterFromCatalog : Command { + public static readonly UnregisterFromCatalog Instance = new(); } } @@ -2048,18 +2046,18 @@ public bool IsReentrantSectionActive(Guid reentrancyId) } } - private class DehydrationContextHolder(SerializerSessionPool sessionPool, Dictionary requestContext) + private class DehydrationContextHolder(SerializerSessionPool sessionPool, Dictionary? requestContext) { public readonly MigrationContext MigrationContext = new(sessionPool); - public readonly Dictionary RequestContext = requestContext; + public readonly Dictionary? RequestContext = requestContext; } - private class MigrateWorkItem(ActivationData activation, Dictionary requestContext, CancellationToken cancellationToken) : WorkItemBase + private class MigrateWorkItem(ActivationData activation, Dictionary? requestContext, CancellationTokenSource cts) : WorkItemBase { public override string Name => "Migrate"; public override IGrainContext GrainContext => activation; - public override void Execute() => activation.StartMigratingAsync(requestContext, cancellationToken).Ignore(); + public override void Execute() => activation.StartMigratingAsync(requestContext, cts).Ignore(); } } diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index be1c5f99c4..d870a46dbc 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -226,13 +226,16 @@ public void UnregisterSystemTarget(ISystemTarget target) public int ActivationCount { get { return activations.Count; } } /// - /// If activation already exists, use it - /// Otherwise, create an activation of an existing grain by reading its state. - /// Return immediately using a dummy that will queue messages. - /// Concurrently start creating and initializing the real activation and replace it when it is ready. + /// If activation already exists, return it. + /// Otherwise, creates a new activation, begins rehydrating it and activating it, then returns it. /// - /// The grain identity - /// Request context data. + /// + /// There is no guarantee about the validity of the activation which is returned. + /// Activations are responsible for handling any messages which they receive. + /// + /// The grain identity. + /// Optional request context data. + /// Optional rehydration context. /// public IGrainContext GetOrCreateActivation( in GrainId grainId, diff --git a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs index 4e0dbb5de3..988c3f43cf 100644 --- a/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs +++ b/src/Orleans.Runtime/Catalog/StatelessWorkerGrainContext.cs @@ -1,3 +1,4 @@ +#nullable enable using System; using System.Collections.Concurrent; using System.Collections.Generic; @@ -28,7 +29,7 @@ internal class StatelessWorkerGrainContext : IGrainContext, IAsyncDisposable, IA private readonly Task _messageLoopTask; #pragma warning restore IDE0052 // Remove unread private members - private GrainReference _grainReference; + private GrainReference? _grainReference; public StatelessWorkerGrainContext( GrainAddress address, @@ -46,7 +47,7 @@ internal class StatelessWorkerGrainContext : IGrainContext, IAsyncDisposable, IA public GrainId GrainId => _address.GrainId; - public object GrainInstance => null; + public object? GrainInstance => null; public ActivationId ActivationId => _address.ActivationId; @@ -70,10 +71,8 @@ public Task Deactivated } } - public void Activate(Dictionary requestContext, CancellationToken? cancellationToken = null) + public void Activate(Dictionary? requestContext, CancellationToken cancellationToken) { - _workItems.Enqueue(new(WorkItemType.Activate, new ActivateWorkItemState(requestContext, cancellationToken))); - _workSignal.Signal(); } public void ReceiveMessage(object message) @@ -82,7 +81,7 @@ public void ReceiveMessage(object message) _workSignal.Signal(); } - public void Deactivate(DeactivationReason deactivationReason, CancellationToken? cancellationToken = null) + public void Deactivate(DeactivationReason deactivationReason, CancellationToken cancellationToken) { _workItems.Enqueue(new(WorkItemType.Deactivate, new DeactivateWorkItemState(deactivationReason, cancellationToken))); _workSignal.Signal(); @@ -97,13 +96,13 @@ public async ValueTask DisposeAsync() public bool Equals([AllowNull] IGrainContext other) => other is not null && ActivationId.Equals(other.ActivationId); - public TComponent GetComponent() where TComponent : class => this switch + public TComponent? GetComponent() where TComponent : class => this switch { TComponent contextResult => contextResult, _ => _shared.GetComponent() }; - public void SetComponent(TComponent instance) where TComponent : class => throw new ArgumentException($"Cannot set a component on a {nameof(StatelessWorkerGrainContext)}"); + public void SetComponent(TComponent? instance) where TComponent : class => throw new ArgumentException($"Cannot set a component on a {nameof(StatelessWorkerGrainContext)}"); public TTarget GetTarget() where TTarget : class => throw new NotImplementedException(); @@ -120,12 +119,6 @@ private async Task RunMessageLoop() case WorkItemType.Message: ReceiveMessageInternal(workItem.State); break; - case WorkItemType.Activate: - { - var state = (ActivateWorkItemState)workItem.State; - ActivateInternal(state.RequestContext, state.CancellationToken); - break; - } case WorkItemType.Deactivate: { var state = (DeactivateWorkItemState)workItem.State; @@ -167,8 +160,8 @@ private void ReceiveMessageInternal(object message) { try { - ActivationData worker = null; - ActivationData minimumWaitingCountWorker = null; + ActivationData? worker = null; + ActivationData? minimumWaitingCountWorker = null; var minimumWaitingCount = int.MaxValue; // Make sure there is at least one worker @@ -200,18 +193,16 @@ private void ReceiveMessageInternal(object message) } } - if (worker == null) + if (worker is null) { - // There are no inactive workers, can we make more of them? - if (_workers.Count < _maxWorkers) - { - worker = CreateWorker(message); - } - else + if (_workers.Count >= _maxWorkers) { // Pick the one with the lowest waiting count worker = minimumWaitingCountWorker; } + + // If there are no workers, make one. + worker ??= CreateWorker(message); } } @@ -244,12 +235,7 @@ private ActivationData CreateWorker(object message) return newWorker; } - private void ActivateInternal(Dictionary requestContext, CancellationToken? cancellationToken) - { - // No-op - } - - private void DeactivateInternal(DeactivationReason reason, CancellationToken? cancellationToken) + private void DeactivateInternal(DeactivationReason reason, CancellationToken cancellationToken) { foreach (var worker in _workers) { @@ -325,23 +311,22 @@ public void Rehydrate(IRehydrationContext context) (context as IDisposable)?.Dispose(); } - public void Migrate(Dictionary requestContext, CancellationToken? cancellationToken = null) + public void Migrate(Dictionary? requestContext, CancellationToken cancellationToken) { // Migration is not supported. Do nothing: the contract is that this method attempts migration, but does not guarantee it will occur. } private enum WorkItemType { - Activate = 0, - Message = 1, - Deactivate = 2, - DeactivatedTask = 3, - DisposeAsync = 4, - OnDestroyActivation = 5, + Message, + Deactivate, + DeactivatedTask, + DisposeAsync, + OnDestroyActivation, } - private record ActivateWorkItemState(Dictionary RequestContext, CancellationToken? CancellationToken); - private record DeactivateWorkItemState(DeactivationReason DeactivationReason, CancellationToken? CancellationToken); + private record ActivateWorkItemState(Dictionary? RequestContext, CancellationToken CancellationToken); + private record DeactivateWorkItemState(DeactivationReason DeactivationReason, CancellationToken CancellationToken); private record DeactivatedTaskWorkItemState(TaskCompletionSource Completion); private record DisposeAsyncWorkItemState(TaskCompletionSource Completion); } diff --git a/src/Orleans.Runtime/Core/GrainRuntime.cs b/src/Orleans.Runtime/Core/GrainRuntime.cs index 454f646a2c..82892e15ae 100644 --- a/src/Orleans.Runtime/Core/GrainRuntime.cs +++ b/src/Orleans.Runtime/Core/GrainRuntime.cs @@ -1,101 +1,106 @@ +#nullable enable using System; using Orleans.Core; using Orleans.Timers; using Orleans.Storage; -namespace Orleans.Runtime +namespace Orleans.Runtime; + +internal class GrainRuntime : IGrainRuntime { - internal class GrainRuntime : IGrainRuntime + private readonly IServiceProvider _serviceProvider; + + private readonly ITimerRegistry _timerRegistry; + private readonly IGrainFactory _grainFactory; + + public GrainRuntime( + ILocalSiloDetails localSiloDetails, + IGrainFactory grainFactory, + ITimerRegistry timerRegistry, + IServiceProvider serviceProvider, + TimeProvider timeProvider) { - private readonly IServiceProvider serviceProvider; - private readonly ITimerRegistry timerRegistry; - private readonly IGrainFactory grainFactory; - - public GrainRuntime( - ILocalSiloDetails localSiloDetails, - IGrainFactory grainFactory, - ITimerRegistry timerRegistry, - IServiceProvider serviceProvider) - { - SiloAddress = localSiloDetails.SiloAddress; - SiloIdentity = SiloAddress.ToString(); - this.grainFactory = grainFactory; - this.timerRegistry = timerRegistry; - this.serviceProvider = serviceProvider; - } + SiloAddress = localSiloDetails.SiloAddress; + SiloIdentity = SiloAddress.ToString(); + _grainFactory = grainFactory; + _timerRegistry = timerRegistry; + _serviceProvider = serviceProvider; + TimeProvider = timeProvider; + } - public string SiloIdentity { get; } + public string SiloIdentity { get; } - public SiloAddress SiloAddress { get; } + public SiloAddress SiloAddress { get; } - public IGrainFactory GrainFactory + public IGrainFactory GrainFactory + { + get { - get - { - CheckRuntimeContext(RuntimeContext.Current); - return this.grainFactory; - } + CheckRuntimeContext(RuntimeContext.Current); + return _grainFactory; } + } - public ITimerRegistry TimerRegistry + public ITimerRegistry TimerRegistry + { + get { - get - { - CheckRuntimeContext(RuntimeContext.Current); - return this.timerRegistry; - } + CheckRuntimeContext(RuntimeContext.Current); + return _timerRegistry; } + } - public IServiceProvider ServiceProvider + public IServiceProvider ServiceProvider + { + get { - get - { - CheckRuntimeContext(RuntimeContext.Current); - return this.serviceProvider; - } + CheckRuntimeContext(RuntimeContext.Current); + return _serviceProvider; } + } - public void DeactivateOnIdle(IGrainContext grainContext) + public TimeProvider TimeProvider { get; } + + public void DeactivateOnIdle(IGrainContext grainContext) + { + CheckRuntimeContext(grainContext); + grainContext.Deactivate(new(DeactivationReasonCode.ApplicationRequested, $"{nameof(DeactivateOnIdle)} was called.")); + } + + public void DelayDeactivation(IGrainContext grainContext, TimeSpan timeSpan) + { + CheckRuntimeContext(grainContext); + if (grainContext is not ICollectibleGrainContext collectibleContext) { - CheckRuntimeContext(grainContext); - grainContext.Deactivate(new(DeactivationReasonCode.ApplicationRequested, $"{nameof(DeactivateOnIdle)} was called.")); + throw new NotSupportedException($"Grain context {grainContext} does not implement {nameof(ICollectibleGrainContext)} and therefore {nameof(DelayDeactivation)} is not supported"); } - public void DelayDeactivation(IGrainContext grainContext, TimeSpan timeSpan) - { - CheckRuntimeContext(grainContext); - if (grainContext is not ICollectibleGrainContext collectibleContext) - { - throw new NotSupportedException($"Grain context {grainContext} does not implement {nameof(ICollectibleGrainContext)} and therefore {nameof(DelayDeactivation)} is not supported"); - } + collectibleContext.DelayDeactivation(timeSpan); + } - collectibleContext.DelayDeactivation(timeSpan); - } + public IStorage GetStorage(IGrainContext grainContext) + { + ArgumentNullException.ThrowIfNull(grainContext); + var grainType = grainContext.GrainInstance?.GetType() ?? throw new ArgumentNullException(nameof(IGrainContext.GrainInstance)); + IGrainStorage grainStorage = GrainStorageHelpers.GetGrainStorage(grainType, ServiceProvider); + return new StateStorageBridge("state", grainContext, grainStorage); + } - public IStorage GetStorage(IGrainContext grainContext) + public static void CheckRuntimeContext(IGrainContext context) + { + if (context is null) { - if (grainContext is null) throw new ArgumentNullException(nameof(grainContext)); - var grainType = grainContext.GrainInstance?.GetType() ?? throw new ArgumentNullException(nameof(IGrainContext.GrainInstance)); - IGrainStorage grainStorage = GrainStorageHelpers.GetGrainStorage(grainType, ServiceProvider); - return new StateStorageBridge("state", grainContext, grainStorage); + // Move exceptions into local functions to help inlining this method. + ThrowMissingContext(); + void ThrowMissingContext() => throw new InvalidOperationException("Activation access violation. A non-activation thread attempted to access activation services."); } - public static void CheckRuntimeContext(IGrainContext context) + if (context is ActivationData activation + && (activation.State == ActivationState.Invalid || activation.State == ActivationState.FailedToActivate)) { - if (context is null) - { - // Move exceptions into local functions to help inlining this method. - ThrowMissingContext(); - void ThrowMissingContext() => throw new InvalidOperationException("Activation access violation. A non-activation thread attempted to access activation services."); - } - - if (context is ActivationData activation - && (activation.State == ActivationState.Invalid || activation.State == ActivationState.FailedToActivate)) - { - // Move exceptions into local functions to help inlining this method. - ThrowInvalidActivation(activation); - void ThrowInvalidActivation(ActivationData activationData) => throw new InvalidOperationException($"Attempt to access an invalid activation: {activationData}"); - } + // Move exceptions into local functions to help inlining this method. + ThrowInvalidActivation(activation); + void ThrowInvalidActivation(ActivationData activationData) => throw new InvalidOperationException($"Attempt to access an invalid activation: {activationData}"); } } } diff --git a/src/Orleans.Runtime/Core/HostedClient.cs b/src/Orleans.Runtime/Core/HostedClient.cs index 825dc619d2..5a2c6f2a1a 100644 --- a/src/Orleans.Runtime/Core/HostedClient.cs +++ b/src/Orleans.Runtime/Core/HostedClient.cs @@ -1,6 +1,8 @@ +#nullable enable using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -31,7 +33,7 @@ internal sealed class HostedClient : IGrainContext, IGrainExtensionBinder, IDisp private readonly ConcurrentDictionary _components = new(); private readonly IServiceScope _serviceProviderScope; private bool disposing; - private Task messagePump; + private Task? messagePump; public HostedClient( IRuntimeClient runtimeClient, @@ -79,7 +81,7 @@ internal sealed class HostedClient : IGrainContext, IGrainExtensionBinder, IDisp public GrainId GrainId => this.ClientId.GrainId; - public object GrainInstance => null; + public object? GrainInstance => null; public ActivationId ActivationId => this.Address.ActivationId; @@ -93,8 +95,6 @@ internal sealed class HostedClient : IGrainContext, IGrainExtensionBinder, IDisp public bool IsExemptFromCollection => true; - public PlacementStrategy PlacementStrategy => null; - /// public override string ToString() => $"{nameof(HostedClient)}_{this.Address}"; @@ -134,7 +134,7 @@ public void DeleteObjectReference(IAddressable obj) } } - public TComponent GetComponent() where TComponent : class + public TComponent? GetComponent() where TComponent : class { if (this is TComponent component) return component; if (_components.TryGetValue(typeof(TComponent), out var result)) @@ -157,7 +157,7 @@ public void DeleteObjectReference(IAddressable obj) return default; } - public void SetComponent(TComponent instance) where TComponent : class + public void SetComponent(TComponent? instance) where TComponent : class { if (this is TComponent) { @@ -292,7 +292,7 @@ async Task OnStop(CancellationToken cancellation) } } - public bool Equals(IGrainContext other) => ReferenceEquals(this, other); + public bool Equals(IGrainContext? other) => ReferenceEquals(this, other); public (TExtension, TExtensionInterface) GetOrSetExtension(Func newExtensionFunc) where TExtension : class, TExtensionInterface @@ -338,7 +338,7 @@ async Task OnStop(CancellationToken cancellation) return false; } - private bool TryGetExtension(out TExtensionInterface result) + private bool TryGetExtension([NotNullWhen(true)] out TExtensionInterface? result) where TExtensionInterface : IGrainExtension { if (_extensions.TryGetValue(typeof(TExtensionInterface), out var existing)) @@ -380,8 +380,8 @@ public TExtensionInterface GetExtension() } public TTarget GetTarget() where TTarget : class => throw new NotImplementedException(); - public void Activate(Dictionary requestContext, CancellationToken? cancellationToken = null) { } - public void Deactivate(DeactivationReason deactivationReason, CancellationToken? cancellationToken = null) { } + public void Activate(Dictionary? requestContext, CancellationToken cancellationToken) { } + public void Deactivate(DeactivationReason deactivationReason, CancellationToken cancellationToken) { } public Task Deactivated => Task.CompletedTask; public void Rehydrate(IRehydrationContext context) @@ -390,7 +390,7 @@ public void Rehydrate(IRehydrationContext context) (context as IDisposable)?.Dispose(); } - public void Migrate(Dictionary requestContext, CancellationToken? cancellationToken = null) + public void Migrate(Dictionary? requestContext, CancellationToken cancellationToken) { // Migration is not supported. Do nothing: the contract is that this method attempts migration, but does not guarantee it will occur. } diff --git a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs index 2082916d44..6ab09e22f9 100644 --- a/src/Orleans.Runtime/Core/InsideRuntimeClient.cs +++ b/src/Orleans.Runtime/Core/InsideRuntimeClient.cs @@ -150,7 +150,7 @@ private List GrainCallFilters sharedData = this.sharedCallbackData; } - if (message.IsExpirableMessage(this.messagingOptions.DropExpiredMessages)) + if (this.messagingOptions.DropExpiredMessages && message.IsExpirableMessage()) { message.TimeToLive = request.GetDefaultResponseTimeout() ?? sharedData.ResponseTimeout; } diff --git a/src/Orleans.Runtime/Core/InternalGrainRuntime.cs b/src/Orleans.Runtime/Core/InternalGrainRuntime.cs index d280fea405..d052aefd73 100644 --- a/src/Orleans.Runtime/Core/InternalGrainRuntime.cs +++ b/src/Orleans.Runtime/Core/InternalGrainRuntime.cs @@ -10,40 +10,26 @@ namespace Orleans.Runtime /// /// Shared runtime services which grains use. /// - internal class InternalGrainRuntime + internal class InternalGrainRuntime( + MessageCenter messageCenter, + Catalog catalog, + GrainVersionManifest versionManifest, + RuntimeMessagingTrace messagingTrace, + GrainLocator grainLocator, + CompatibilityDirectorManager compatibilityDirectorManager, + IOptions collectionOptions, + ILocalGrainDirectory localGrainDirectory, + IActivationWorkingSet activationWorkingSet) { - public InternalGrainRuntime( - MessageCenter messageCenter, - Catalog catalog, - GrainVersionManifest versionManifest, - RuntimeMessagingTrace messagingTrace, - GrainLocator grainLocator, - CompatibilityDirectorManager compatibilityDirectorManager, - IOptions collectionOptions, - ILocalGrainDirectory localGrainDirectory, - IActivationWorkingSet activationWorkingSet) - { - MessageCenter = messageCenter; - Catalog = catalog; - RuntimeClient = catalog.RuntimeClient; - GrainVersionManifest = versionManifest; - MessagingTrace = messagingTrace; - CompatibilityDirectorManager = compatibilityDirectorManager; - GrainLocator = grainLocator; - CollectionOptions = collectionOptions; - LocalGrainDirectory = localGrainDirectory; - ActivationWorkingSet = activationWorkingSet; - } - - public InsideRuntimeClient RuntimeClient { get; } - public MessageCenter MessageCenter { get; } - public Catalog Catalog { get; } - public GrainVersionManifest GrainVersionManifest { get; } - public RuntimeMessagingTrace MessagingTrace { get; } - public CompatibilityDirectorManager CompatibilityDirectorManager { get; } - public GrainLocator GrainLocator { get; } - public IOptions CollectionOptions { get; } - public ILocalGrainDirectory LocalGrainDirectory { get; } - public IActivationWorkingSet ActivationWorkingSet { get; } + public InsideRuntimeClient RuntimeClient { get; } = catalog.RuntimeClient; + public MessageCenter MessageCenter { get; } = messageCenter; + public Catalog Catalog { get; } = catalog; + public GrainVersionManifest GrainVersionManifest { get; } = versionManifest; + public RuntimeMessagingTrace MessagingTrace { get; } = messagingTrace; + public CompatibilityDirectorManager CompatibilityDirectorManager { get; } = compatibilityDirectorManager; + public GrainLocator GrainLocator { get; } = grainLocator; + public IOptions CollectionOptions { get; } = collectionOptions; + public ILocalGrainDirectory LocalGrainDirectory { get; } = localGrainDirectory; + public IActivationWorkingSet ActivationWorkingSet { get; } = activationWorkingSet; } } diff --git a/src/Orleans.Runtime/Core/SystemTarget.cs b/src/Orleans.Runtime/Core/SystemTarget.cs index 7b1d209758..c06a9b8db0 100644 --- a/src/Orleans.Runtime/Core/SystemTarget.cs +++ b/src/Orleans.Runtime/Core/SystemTarget.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -16,9 +17,10 @@ namespace Orleans.Runtime /// Made public for GrainService to inherit from it. /// Can be turned to internal after a refactoring that would remove the inheritance relation. /// - public abstract class SystemTarget : ISystemTarget, ISystemTargetBase, IGrainContext, IGrainExtensionBinder, ISpanFormattable, IDisposable + public abstract class SystemTarget : ISystemTarget, ISystemTargetBase, IGrainContext, IGrainExtensionBinder, ISpanFormattable, IDisposable, IGrainTimerRegistry { private readonly SystemTargetGrainId id; + private readonly HashSet _timers = []; private GrainReference selfReference; private Message running; private Dictionary _components = new Dictionary(); @@ -30,7 +32,6 @@ public abstract class SystemTarget : ISystemTarget, ISystemTargetBase, IGrainCon internal ActivationId ActivationId { get; set; } private InsideRuntimeClient runtimeClient; private RuntimeMessagingTrace messagingTrace; - private readonly ILogger timerLogger; private readonly ILogger logger; internal InsideRuntimeClient RuntimeClient @@ -67,8 +68,8 @@ protected SystemTarget() { } - internal SystemTarget(GrainType grainType, SiloAddress silo, ILoggerFactory loggerFactory) - : this(SystemTargetGrainId.Create(grainType, silo), silo, loggerFactory) + internal SystemTarget(GrainType grainType, SiloAddress siloAddress, ILoggerFactory loggerFactory) + : this(SystemTargetGrainId.Create(grainType, siloAddress), siloAddress, loggerFactory) { } @@ -78,7 +79,6 @@ internal SystemTarget(SystemTargetGrainId grainId, SiloAddress silo, ILoggerFact this.Silo = silo; this.ActivationId = ActivationId.GetDeterministic(grainId.GrainId); this.ActivationAddress = GrainAddress.GetAddress(this.Silo, this.id.GrainId, this.ActivationId); - this.timerLogger = loggerFactory.CreateLogger(); this.logger = loggerFactory.CreateLogger(this.GetType()); if (!Constants.IsSingletonSystemTarget(GrainId.Type)) @@ -160,32 +160,26 @@ internal void HandleResponse(Message response) /// Registers a timer to send regular callbacks to this grain. /// This timer will keep the current grain from being deactivated. /// - /// The timer callback, which will fire whenever the timer becomes due. + /// The timer callback, which will fire whenever the timer becomes due. /// The state object passed to the callback. /// - /// The amount of time to delay before the is invoked. + /// The amount of time to delay before the is invoked. /// Specify to prevent the timer from starting. /// Specify to invoke the callback promptly. /// /// - /// The time interval between invocations of . + /// The time interval between invocations of . /// Specify to disable periodic signaling. /// /// /// An object which will cancel the timer upon disposal. /// - public IGrainTimer RegisterTimer(Func asyncCallback, object state, TimeSpan dueTime, TimeSpan period) - => RegisterGrainTimer(asyncCallback, state, dueTime, period); - - /// - /// Internal version of that returns the inner IGrainTimer - /// - internal IGrainTimer RegisterGrainTimer(Func asyncCallback, object state, TimeSpan dueTime, TimeSpan period) + public IGrainTimer RegisterTimer(Func callback, object state, TimeSpan dueTime, TimeSpan period) { var ctxt = RuntimeContext.Current; - - var timer = new GrainTimer(this, this.timerLogger, asyncCallback, state, RuntimeClient.TimeProvider); - timer.Change(dueTime, period); + ArgumentNullException.ThrowIfNull(callback); + var timer = this.ActivationServices.GetRequiredService() + .RegisterGrainTimer(this, static (state, _) => state.Callback(state.State), (Callback: callback, State: state), new() { DueTime = dueTime, Period = period, Interleave = true }); return timer; } @@ -293,10 +287,10 @@ public void ReceiveMessage(object message) public TTarget GetTarget() where TTarget : class => (TTarget)(object)this; /// - public void Activate(Dictionary requestContext, CancellationToken? cancellationToken = null) { } + public void Activate(Dictionary requestContext, CancellationToken cancellationToken) { } /// - public void Deactivate(DeactivationReason deactivationReason, CancellationToken? cancellationToken = null) { } + public void Deactivate(DeactivationReason deactivationReason, CancellationToken cancellationToken) { } /// public Task Deactivated => Task.CompletedTask; @@ -307,6 +301,8 @@ public void Dispose() { GrainInstruments.DecrementSystemTargetCounts(Constants.SystemTargetName(GrainId.Type)); } + + StopAllTimers(); } public void Rehydrate(IRehydrationContext context) @@ -315,9 +311,26 @@ public void Rehydrate(IRehydrationContext context) (context as IDisposable)?.Dispose(); } - public void Migrate(Dictionary requestContext, CancellationToken? cancellationToken = null) + public void Migrate(Dictionary requestContext, CancellationToken cancellationToken) { // Migration is not supported. Do nothing: the contract is that this method attempts migration, but does not guarantee it will occur. } + + void IGrainTimerRegistry.OnTimerCreated(IGrainTimer timer) { lock (_timers) { _timers.Add(timer); } } + void IGrainTimerRegistry.OnTimerDisposed(IGrainTimer timer) { lock (_timers) { _timers.Remove(timer); } } + private void StopAllTimers() + { + List timers; + lock (_timers) + { + timers = _timers.ToList(); + _timers.Clear(); + } + + foreach (var timer in timers) + { + timer.Dispose(); + } + } } } diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index 6d92dd175c..f814a109db 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -115,7 +115,6 @@ internal static void AddDefaultServices(ISiloBuilder builder) services.TryAddSingleton(); services.TryAddSingleton(); services.AddSingleton(); - services.AddFromExisting(); services.AddFromExisting(); services.AddFromExisting, ActivationCollector>(); diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index e095ddc4ba..96564a950f 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -10,11 +11,6 @@ namespace Orleans.Runtime.Messaging { - internal interface IMessagingSystemTarget : ISystemTarget - { - ValueTask OnMessagesForwarded(List<(GrainId TargetGrainId, CorrelationId CorrelationId, SiloAddress From, SiloAddress To)> forwards); - } - internal class MessageCenter : IMessageCenter, IAsyncDisposable { private readonly ISiloStatusOracle siloStatusOracle; @@ -136,6 +132,8 @@ public Action SniffIncomingMessage public void SendMessage(Message msg) { + Debug.Assert(!msg.IsLocalOnly); + // Note that if we identify or add other grains that are required for proper stopping, we will need to treat them as we do the membership table grain here. if (IsBlockingApplicationMessages && !msg.IsSystemMessage && msg.Result is not Message.ResponseTypes.Rejection && !Constants.SystemMembershipTableType.Equals(msg.TargetGrain)) { @@ -282,6 +280,8 @@ static async Task SendAsync(MessageCenter messageCenter, ValueTask c foreach (var message in messages) { + Debug.Assert(!message.IsLocalOnly); + if (oldAddress != null) { message.AddToCacheInvalidationHeader(oldAddress, validAddress: validAddress); @@ -310,7 +310,7 @@ static async Task SendAsync(MessageCenter messageCenter, ValueTask c } } - internal void ProcessRequestToInvalidActivation( + private void ProcessRequestToInvalidActivation( Message message, GrainAddress oldAddress, SiloAddress forwardingAddress, @@ -318,6 +318,8 @@ static async Task SendAsync(MessageCenter messageCenter, ValueTask c Exception exc = null, bool rejectMessages = false) { + Debug.Assert(!message.IsLocalOnly); + // Just use this opportunity to invalidate local Cache Entry as well. if (oldAddress != null) { @@ -344,8 +346,10 @@ static async Task SendAsync(MessageCenter messageCenter, ValueTask c } } - internal void TryForwardRequest(Message message, GrainAddress oldAddress, GrainAddress destination, string failedOperation = null, Exception exc = null) + private void TryForwardRequest(Message message, GrainAddress oldAddress, GrainAddress destination, string failedOperation = null, Exception exc = null) { + Debug.Assert(!message.IsLocalOnly); + bool forwardingSucceeded = false; var forwardingAddress = destination?.SiloAddress; try @@ -400,7 +404,7 @@ internal void RerouteMessage(Message message) ResendMessageImpl(message); } - internal bool TryForwardMessage(Message message, SiloAddress forwardingAddress) + private bool TryForwardMessage(Message message, SiloAddress forwardingAddress) { if (!MayForward(message, this.messagingOptions)) return false; @@ -502,6 +506,7 @@ internal void SendResponse(Message request, Response response) public void ReceiveMessage(Message msg) { + Debug.Assert(!msg.IsLocalOnly); try { this.messagingTrace.OnIncomingMessageAgentReceiveMessage(msg); diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index 1f6c53a1a2..3428b7905b 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -84,7 +85,7 @@ public Task AddressMessage(Message message) if (_logger.IsEnabled(LogLevel.Debug)) { - _logger.LogDebug("Placing grain {GrainId} for message {Message}", grainId, message); + _logger.LogDebug("Looking up address for grain {GrainId} for message {Message}", grainId, message); } var worker = _workers[grainId.GetUniformHashCode() % PlacementWorkerCount]; @@ -275,11 +276,7 @@ private async Task ProcessLoop() foreach (var message in messages) { var target = message.Message.TargetGrain; - if (!_inProgress.TryGetValue(target, out var workItem)) - { - _inProgress[target] = workItem = new(); - } - + var workItem = GetOrAddWorkItem(target); workItem.Messages.Add(message); if (workItem.Result is null) { @@ -308,11 +305,18 @@ private async Task ProcessLoop() } catch (Exception exception) { - _logger.LogWarning(exception, "Exception in placement worker"); + _logger.LogWarning(exception, "Error in placement worker."); } await _workSignal.WaitAsync(); } + + GrainPlacementWorkItem GetOrAddWorkItem(GrainId target) + { + ref var workItem = ref CollectionsMarshal.GetValueRefOrAddDefault(_inProgress, target, out _); + workItem ??= new(); + return workItem; + } } private void AddressWaitingMessages(GrainPlacementWorkItem completedWorkItem) diff --git a/src/Orleans.Runtime/Silo/Silo.cs b/src/Orleans.Runtime/Silo/Silo.cs index 7d983801be..29095b4e43 100644 --- a/src/Orleans.Runtime/Silo/Silo.cs +++ b/src/Orleans.Runtime/Silo/Silo.cs @@ -536,11 +536,18 @@ private async Task OnBecomeActiveStop(CancellationToken ct) try { - await catalog.DeactivateAllActivations().WithCancellation(ct); + await catalog.DeactivateAllActivations().WaitAsync(ct); } catch (Exception exception) { - logger.LogError(exception, "Error deactivating activations"); + if (!ct.IsCancellationRequested) + { + logger.LogError(exception, "Error deactivating activations."); + } + else + { + logger.LogWarning("Some grains failed to deactivate promptly."); + } } // Wait for all queued message sent to OutboundMessageQueue before MessageCenter stop and OutboundMessageQueue stop. diff --git a/src/Orleans.Runtime/Timers/GrainTimer.cs b/src/Orleans.Runtime/Timers/GrainTimer.cs index 09f67cde58..31765a6395 100644 --- a/src/Orleans.Runtime/Timers/GrainTimer.cs +++ b/src/Orleans.Runtime/Timers/GrainTimer.cs @@ -1,57 +1,58 @@ #nullable enable using System; using System.Diagnostics.CodeAnalysis; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Orleans.CodeGeneration; using Orleans.Runtime.Internal; +using Orleans.Serialization.Invocation; +using Orleans.Timers; namespace Orleans.Runtime; -internal sealed class GrainTimer : IGrainTimer, IAsyncDisposable +internal abstract class GrainTimer : IGrainTimer { - // PeriodicTimer only supports periods equal to -1ms (infinite timeout) or >= 1ms - private static readonly TimeSpan MinimumPeriod = TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond); - private readonly PeriodicTimer _timer; - private readonly Func _callback; - private readonly ILogger _logger; + protected static readonly GrainInterfaceType InvokableInterfaceType = GrainInterfaceType.Create("Orleans.Runtime.IGrainTimerInvoker"); + protected static readonly TimerCallback TimerCallback = (state) => ((GrainTimer)state!).ScheduleTickOnActivation(); + protected static readonly MethodInfo InvokableMethodInfo = typeof(IGrainTimerInvoker).GetMethod(nameof(IGrainTimerInvoker.InvokeCallbackAsync), BindingFlags.Instance | BindingFlags.Public)!; + private readonly CancellationTokenSource _cts = new(); + private readonly ITimer _timer; private readonly IGrainContext _grainContext; - private readonly Task _processingTask; - private readonly object? _state; + private readonly TimerRegistry _shared; + private readonly bool _interleave; + private readonly bool _keepAlive; + private readonly TimerTickInvoker _invoker; + private bool _changed; + private bool _firing; private TimeSpan _dueTime; private TimeSpan _period; - private bool _changed; - public GrainTimer(IGrainContext grainContext, ILogger logger, Func callback, object? state, TimeProvider timeProvider) + public GrainTimer(TimerRegistry shared, IGrainContext grainContext, bool interleave, bool keepAlive) { + ArgumentNullException.ThrowIfNull(shared); ArgumentNullException.ThrowIfNull(grainContext); - ArgumentNullException.ThrowIfNull(logger); - ArgumentNullException.ThrowIfNull(callback); - if (RuntimeContext.Current is null) - { - ThrowInvalidSchedulingContext(); - } - - if (!Equals(RuntimeContext.Current, grainContext)) - { - ThrowIncorrectGrainContext(); - } + _interleave = interleave; + _keepAlive = keepAlive; + _shared = shared; + _grainContext = grainContext; + _dueTime = Timeout.InfiniteTimeSpan; + _period = Timeout.InfiniteTimeSpan; + _invoker = new(this); // Avoid capturing async locals. using (new ExecutionContextSuppressor()) { - _grainContext = grainContext; - _logger = logger; - _callback = callback; - _timer = new PeriodicTimer(Timeout.InfiniteTimeSpan, timeProvider); - _state = state; - _dueTime = Timeout.InfiniteTimeSpan; - _period = Timeout.InfiniteTimeSpan; - _processingTask = ProcessTimerTicks(); + _timer = shared.TimeProvider.CreateTimer(TimerCallback, this, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); } } + protected IGrainContext GrainContext => _grainContext; + + private ILogger Logger => _shared.TimerLogger; + [DoesNotReturn] private static void ThrowIncorrectGrainContext() => throw new InvalidOperationException("Current grain context differs from specified grain context."); @@ -64,49 +65,135 @@ private static void ThrowInvalidSchedulingContext() + "which will be the case if you create it inside Task.Run."); } - private async Task ProcessTimerTicks() + protected void ScheduleTickOnActivation() { - // Yield immediately to let the caller continue. - await Task.Yield(); + try + { + // Indicate that the timer is firing so that the effect of the next change call is deferred until after the tick completes. + _firing = true; + + // Note: this does not execute on the activation's execution context. + var msg = _shared.MessageFactory.CreateMessage(body: _invoker, options: InvokeMethodOptions.OneWay); + msg.SetInfiniteTimeToLive(); + msg.SendingGrain = _grainContext.GrainId; + msg.TargetGrain = _grainContext.GrainId; + msg.SendingSilo = _shared.LocalSiloDetails.SiloAddress; + msg.TargetSilo = _shared.LocalSiloDetails.SiloAddress; + msg.InterfaceType = InvokableInterfaceType; + msg.IsKeepAlive = _keepAlive; + msg.IsAlwaysInterleave = _interleave; + + // Prevent the message from being forwarded in the case of deactivation. + msg.IsLocalOnly = true; - while (await _timer.WaitForNextTickAsync()) + _grainContext.ReceiveMessage(msg); + } + catch (Exception exception) { - _changed = false; try { - if (_logger.IsEnabled(LogLevel.Trace)) - { - _logger.LogTrace((int)ErrorCode.TimerBeforeCallback, "About to make timer callback for timer {TimerName}", GetDiagnosticName()); - } + Logger.LogError(exception, "Error invoking timer tick for timer '{Timer}'.", this); + } + catch + { + // Ignore. + // Allowing an exception to escape here would crash the process. + } + } + } - await _callback(_state); + protected abstract Task InvokeCallbackAsync(CancellationToken cancellationToken); - if (_logger.IsEnabled(LogLevel.Trace)) + private ValueTask InvokeGrainTimerCallbackAsync() + { + try + { + if (Logger.IsEnabled(LogLevel.Trace)) + { + Logger.LogTrace((int)ErrorCode.TimerBeforeCallback, "About to invoke callback for timer {Timer}", this); + } + + _changed = false; + var task = InvokeCallbackAsync(_cts.Token); + + // If the task is not completed, we need to await the tick asynchronously. + if (task is { IsCompletedSuccessfully: false }) + { + // Complete asynchronously. + return AwaitCallbackTask(task); + } + else + { + // Complete synchronously. + if (Logger.IsEnabled(LogLevel.Trace)) { - _logger.LogTrace((int)ErrorCode.TimerAfterCallback, "Completed timer callback for timer {TimerName}", GetDiagnosticName()); + Logger.LogTrace((int)ErrorCode.TimerAfterCallback, "Completed timer callback for timer {Timer}", this); } + + OnTickCompleted(); + return new(Response.Completed); } - catch (Exception exc) + } + catch (Exception exc) + { + OnTickCompleted(); + return new(OnCallbackException(exc)); + } + } + + private void OnTickCompleted() + { + // Schedule the next tick. + try + { + if (!_changed) { - _logger.LogError( - (int)ErrorCode.Timer_GrainTimerCallbackError, - exc, - "Caught and ignored exception thrown from timer callback for timer {TimerName}", - GetDiagnosticName()); + // If the timer was not modified during the tick, schedule the next tick based on the period. + _timer.Change(_period, Timeout.InfiniteTimeSpan); } + else + { + // If the timer was modified during the tick, schedule the next tick based on the new due time. + _timer.Change(_dueTime, Timeout.InfiniteTimeSpan); + } + } + catch (ObjectDisposedException) + { + } + + _firing = false; + } + + private Response OnCallbackException(Exception exc) + { + Logger.LogWarning( + (int)ErrorCode.Timer_GrainTimerCallbackError, + exc, + "Caught and ignored exception thrown from timer callback for timer '{Timer}'.", + this); + return Response.FromException(exc); + } - // Resume regular ticking if the period was not changed during the iteration. - if (!_changed && _timer.Period != _period) + private async ValueTask AwaitCallbackTask(Task task) + { + try + { + await task; + + if (Logger.IsEnabled(LogLevel.Trace)) { - try - { - _timer.Period = _period; - } - catch (ObjectDisposedException) - { - return; - } + Logger.LogTrace((int)ErrorCode.TimerAfterCallback, "Completed timer callback for timer '{Timer}'.", this); } + + return Response.Completed; + } + catch (Exception exc) + { + return OnCallbackException(exc); + } + finally + { + OnTickCompleted(); } } @@ -115,21 +202,22 @@ public void Change(TimeSpan dueTime, TimeSpan period) ValidateArguments(dueTime, period); _changed = true; - _dueTime = AdjustPeriod(dueTime); - _period = AdjustPeriod(period); - _timer.Period = _dueTime; + _dueTime = dueTime; + _period = period; - static TimeSpan AdjustPeriod(TimeSpan value) + // If the timer is currently firing, the change will be deferred until after the tick completes. + // Otherwise, perform the change now. + if (!_firing) { - // Period must be either -1ms (infinite timeout) or >= 1ms - if (value != Timeout.InfiniteTimeSpan && value <= MinimumPeriod) + try + { + // This method resets the timer, so the next tick will be scheduled at the new due time and subsequent + // ticks will be scheduled after the specified period. + _timer.Change(dueTime, Timeout.InfiniteTimeSpan); + } + catch (ObjectDisposedException) { - // Adjust period to 1ms if it is out of bounds. In practice, - // this is smaller than the timer resolution, so the difference is imperceptible. - return MinimumPeriod; } - - return value; } } @@ -148,17 +236,128 @@ private static void ValidateArguments(TimeSpan dueTime, TimeSpan period) ArgumentOutOfRangeException.ThrowIfGreaterThan(periodTm, MaxSupportedTimeout, nameof(period)); } - private string GetDiagnosticName() => $"GrainTimer TimerCallbackHandler:{_callback?.Target}->{_callback?.Method}"; - public void Dispose() { + try + { + _cts.Cancel(); + } + catch (Exception exception) + { + Logger.LogWarning(exception, "Error cancelling timer callback."); + } + _timer.Dispose(); - _grainContext.GetComponent()?.OnTimerDisposed(this); + var timerRegistry = _grainContext.GetComponent(); + timerRegistry?.OnTimerDisposed(this); } - async ValueTask IAsyncDisposable.DisposeAsync() + public override string ToString() => $"[{GetType()}] Grain: '{_grainContext}'"; + + private sealed class TimerTickInvoker(GrainTimer timer) : IInvokable, IGrainTimerInvoker { - Dispose(); - await _processingTask; + public object? GetTarget() => this; + + public void SetTarget(ITargetHolder holder) + { + if (timer._grainContext != holder) + { + throw new InvalidOperationException($"Invalid target holder. Expected {timer._grainContext}, received {holder}."); + } + } + + public ValueTask Invoke() => timer.InvokeGrainTimerCallbackAsync(); + + // This method is declared for the sake of IGrainTimerCore, but it is not intended to be called directly. + // It exists for grain call interceptors which inspect the implementation method. + Task IGrainTimerInvoker.InvokeCallbackAsync() => throw new InvalidOperationException(); + + public int GetArgumentCount() => 0; + + public object? GetArgument(int index) => throw new IndexOutOfRangeException(); + + public void SetArgument(int index, object value) => throw new IndexOutOfRangeException(); + + public string GetMethodName() => nameof(IGrainTimerInvoker.InvokeCallbackAsync); + + public string GetInterfaceName() => nameof(IGrainTimerInvoker); + + public string GetActivityName() => $"{nameof(IGrainTimerInvoker)}/{nameof(IGrainTimerInvoker.InvokeCallbackAsync)}"; + + public MethodInfo GetMethod() => InvokableMethodInfo; + + public Type GetInterfaceType() => typeof(IGrainTimerInvoker); + + public TimeSpan? GetDefaultResponseTimeout() => null; + + public void Dispose() + { + // Do nothing. Instances are disposed after invocation, but this instance will be reused for the lifetime of the timer. + } + + public override string ToString() => timer.ToString(); } } + +internal sealed class GrainTimer : GrainTimer +{ + private readonly Func _callback; + private readonly T _state; + + public GrainTimer( + TimerRegistry shared, + IGrainContext grainContext, + Func callback, + T state, + bool interleave, + bool keepAlive) + : base( + shared, + grainContext, + interleave, + keepAlive) + { + ArgumentNullException.ThrowIfNull(callback); + _callback = callback; + _state = state; + } + + protected override Task InvokeCallbackAsync(CancellationToken cancellationToken) => _callback(_state, cancellationToken); + + public override string ToString() => $"{base.ToString()} Callback: '{_callback?.Target}.{_callback?.Method}'. State: '{_state}'"; +} + +internal sealed class InterleavingGrainTimer : GrainTimer +{ + private readonly Func _callback; + private readonly object? _state; + + public InterleavingGrainTimer( + TimerRegistry shared, + IGrainContext grainContext, + Func callback, + object? state) + : base( + shared, + grainContext, + interleave: true, + keepAlive: false) + { + ArgumentNullException.ThrowIfNull(callback); + _callback = callback; + _state = state; + } + + protected override Task InvokeCallbackAsync(CancellationToken cancellationToken) => _callback(_state); + + public override string ToString() => $"{base.ToString()} Callback: '{_callback?.Target}.{_callback?.Method}'. State: '{_state}'"; +} + +// This interface exists for the IInvokable implementation, so that call filters behave as intended. +internal interface IGrainTimerInvoker : IAddressable +{ + /// + /// Invokes the callback. + /// + Task InvokeCallbackAsync(); +} diff --git a/src/Orleans.Runtime/Timers/IGrainTimerRegistry.cs b/src/Orleans.Runtime/Timers/IGrainTimerRegistry.cs new file mode 100644 index 0000000000..1cd0e5252d --- /dev/null +++ b/src/Orleans.Runtime/Timers/IGrainTimerRegistry.cs @@ -0,0 +1,25 @@ +#nullable enable +namespace Orleans.Runtime; + +/// +/// Provides functionality to record the creation and deletion of grain timers. +/// +internal interface IGrainTimerRegistry +{ + /// + /// Signals to the registry that a timer was created. + /// + /// + /// The timer. + /// + void OnTimerCreated(IGrainTimer timer); + + /// + /// Signals to the registry that a timer was disposed. + /// + /// + /// The timer. + /// + void OnTimerDisposed(IGrainTimer timer); +} + diff --git a/src/Orleans.Runtime/Timers/TimerRegistry.cs b/src/Orleans.Runtime/Timers/TimerRegistry.cs index 4c06f84b6f..dac6069992 100644 --- a/src/Orleans.Runtime/Timers/TimerRegistry.cs +++ b/src/Orleans.Runtime/Timers/TimerRegistry.cs @@ -1,20 +1,36 @@ +#nullable enable using System; +using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Orleans.Runtime; namespace Orleans.Timers; -internal class TimerRegistry(ILoggerFactory loggerFactory, TimeProvider timeProvider) : ITimerRegistry +internal class TimerRegistry(ILoggerFactory loggerFactory, TimeProvider timeProvider, MessageFactory messageFactory, ILocalSiloDetails localSiloDetails) : ITimerRegistry { - private readonly ILogger _timerLogger = loggerFactory.CreateLogger(); - private readonly TimeProvider _timeProvider = timeProvider; + public ILogger TimerLogger { get; } = loggerFactory.CreateLogger(); + public TimeProvider TimeProvider { get; } = timeProvider; + public MessageFactory MessageFactory { get; } = messageFactory; + public ILocalSiloDetails LocalSiloDetails { get; } = localSiloDetails; - public IDisposable RegisterTimer(IGrainContext grainContext, Func asyncCallback, object state, TimeSpan dueTime, TimeSpan period) + public IDisposable RegisterTimer(IGrainContext grainContext, Func callback, object? state, TimeSpan dueTime, TimeSpan period) { - var timer = new GrainTimer(grainContext, _timerLogger, asyncCallback, state, _timeProvider); - grainContext?.GetComponent().OnTimerCreated(timer); + ArgumentNullException.ThrowIfNull(grainContext); + ArgumentNullException.ThrowIfNull(callback); + var timer = new InterleavingGrainTimer(this, grainContext, callback, state); + grainContext.GetComponent()?.OnTimerCreated(timer); timer.Change(dueTime, period); return timer; } + + public IGrainTimer RegisterGrainTimer(IGrainContext grainContext, Func callback, T state, GrainTimerCreationOptions options) + { + ArgumentNullException.ThrowIfNull(grainContext); + ArgumentNullException.ThrowIfNull(callback); + var timer = new GrainTimer(this, grainContext, callback, state, options.Interleave, options.KeepAlive); + grainContext.GetComponent()?.OnTimerCreated(timer); + timer.Change(options.DueTime, options.Period); + return timer; + } } diff --git a/src/Orleans.Serialization/Invocation/ITargetHolder.cs b/src/Orleans.Serialization/Invocation/ITargetHolder.cs index 359d4f0836..7184b27899 100644 --- a/src/Orleans.Serialization/Invocation/ITargetHolder.cs +++ b/src/Orleans.Serialization/Invocation/ITargetHolder.cs @@ -1,22 +1,22 @@ -namespace Orleans.Serialization.Invocation +#nullable enable +namespace Orleans.Serialization.Invocation; + +/// +/// Represents an object which holds an invocation target as well as target extensions. +/// +public interface ITargetHolder { /// - /// Represents an object which holds an invocation target as well as target extensions. + /// Gets the target. /// - public interface ITargetHolder - { - /// - /// Gets the target. - /// - /// The target type. - /// The target. - TTarget GetTarget() where TTarget : class; + /// The target type. + /// The target. + TTarget? GetTarget() where TTarget : class; - /// - /// Gets the component with the specified type. - /// - /// The component type. - /// The component with the specified type. - TComponent GetComponent() where TComponent : class; - } + /// + /// Gets the component with the specified type. + /// + /// The component type. + /// The component with the specified type. + TComponent? GetComponent() where TComponent : class; } \ No newline at end of file diff --git a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs index c8a4bba91f..19c8af6bfa 100644 --- a/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs +++ b/src/Orleans.Streaming/PersistentStreams/PersistentStreamPullingAgent.cs @@ -9,7 +9,6 @@ using Orleans.Internal; using Orleans.Runtime; using Orleans.Streams.Filtering; -using Orleans.Timers; namespace Orleans.Streams { @@ -144,7 +143,7 @@ private void InitializeInternal() // Setup a reader for a new receiver. // Even if the receiver failed to initialize, treat it as OK and start pumping it. It's receiver responsibility to retry initialization. var randomTimerOffset = RandomTimeSpan.Next(this.options.GetQueueMsgsTimerPeriod); - timer = RegisterGrainTimer(AsyncTimerCallback, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod); + timer = RegisterTimer(AsyncTimerCallback, QueueId, randomTimerOffset, this.options.GetQueueMsgsTimerPeriod); StreamInstruments.RegisterPersistentStreamPubSubCacheSizeObserve(() => new Measurement(pubSubCache.Count, new KeyValuePair("name", StatisticUniquePostfix))); @@ -158,44 +157,15 @@ public async Task Shutdown() var asyncTimer = timer; timer = null; - if (asyncTimer != null) - { - try - { - if (asyncTimer is IAsyncDisposable asyncDisposable) - { - var task = asyncDisposable.DisposeAsync(); - if (!task.IsCompletedSuccessfully) - { - await task.AsTask().WithTimeout(TimeSpan.FromSeconds(5)); - } - } - else - { - asyncTimer.Dispose(); - } - } - catch (Exception ex) - { - this.logger.LogWarning(ex, "Waiting for the last timer tick failed"); - } - } + asyncTimer.Dispose(); this.queueCache = null; Task localReceiverInitTask = receiverInitTask; if (localReceiverInitTask != null) { - try - { - await localReceiverInitTask; - receiverInitTask = null; - } - catch (Exception) - { - receiverInitTask = null; - // squelch - } + await localReceiverInitTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + receiverInitTask = null; } try @@ -390,7 +360,7 @@ private async Task AsyncTimerCallback(object state) if (IsShutdown) return; // timer was already removed, last tick // loop through the queue until it is empty. - while (!IsShutdown) // timer will be set to null when we are asked to shudown. + while (!IsShutdown) // timer will be set to null when we are asked to shutdown. { int maxCacheAddCount = queueCache?.GetMaxAddCount() ?? QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG; if (maxCacheAddCount != QueueAdapterConstants.UNLIMITED_GET_QUEUE_MSG && maxCacheAddCount <= 0) @@ -404,7 +374,7 @@ private async Task AsyncTimerCallback(object state) i => ReadFromQueue(queueId, receiver, maxCacheAddCount), ReadLoopRetryMax, ReadLoopRetryExceptionFilter, - Constants.INFINITE_TIMESPAN, + Timeout.InfiniteTimeSpan, ReadLoopBackoff); if (!moreData) return; @@ -847,7 +817,7 @@ private async Task RegisterAsStreamProducer(QualifiedStreamId streamId, StreamSe await PubsubRegisterProducer(pubSub, streamId, GrainId, logger); }, AsyncExecutorWithRetries.INFINITE_RETRIES, (exception, i) => !IsShutdown, - Constants.INFINITE_TIMESPAN, + Timeout.InfiniteTimeSpan, DeliveryBackoffProvider); diff --git a/test/DefaultCluster.Tests/TimerOrleansTest.cs b/test/DefaultCluster.Tests/TimerOrleansTest.cs index ee349f2306..0e01fbea60 100644 --- a/test/DefaultCluster.Tests/TimerOrleansTest.cs +++ b/test/DefaultCluster.Tests/TimerOrleansTest.cs @@ -176,6 +176,62 @@ public async Task AsyncTimerTest_GrainCall() } } + [Fact, TestCategory("BVT"), TestCategory("Timers")] + public async Task GrainTimer_TestAllOverloads() + { + var grain = GrainFactory.GetGrain(GetRandomGrainId()); + + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var numTimers = await grain.TestAllTimerOverloads(); + while (true) + { + var completedTimers = await grain.PollCompletedTimers().WaitAsync(cts.Token); + if (completedTimers == numTimers) + { + break; + } + + await Task.Delay(TimeSpan.FromMilliseconds(50), cts.Token); + } + + await grain.TestCompletedTimerResults(); + } + + [Fact, TestCategory("SlowBVT"), TestCategory("Timers")] + public async Task NonReentrantGrainTimer_Test() + { + const string testName = "NonReentrantGrainTimer_Test"; + var delay = TimeSpan.FromSeconds(5); + var wait = delay.Multiply(2); + + var grain = GrainFactory.GetGrain(GetRandomGrainId()); + + // Schedule multiple timers with the same delay + await grain.StartTimer(testName, delay); + await grain.StartTimer($"{testName}_1", delay); + await grain.StartTimer($"{testName}_2", delay); + + // Invoke some non-interleaving methods. + var externalTicks = 0; + var stopwatch = Stopwatch.StartNew(); + while (stopwatch.Elapsed < wait) + { + await grain.ExternalTick("external"); + externalTicks++; + } + + var tickCount = await grain.GetTickCount(); + + Assert.Equal(3 + externalTicks, tickCount); + + var err = await grain.GetException(); + Assert.Null(err); // Should be no exceptions during timer callback + + await grain.StopTimer(testName); + await grain.StopTimer($"{testName}_1"); + await grain.StopTimer($"{testName}_2"); + } + [Fact, TestCategory("SlowBVT"), TestCategory("Timers")] public async Task GrainTimer_Change() { diff --git a/test/Extensions/ServiceBus.Tests/Streaming/EHProgrammaticSubscribeTests.cs b/test/Extensions/ServiceBus.Tests/Streaming/EHProgrammaticSubscribeTests.cs index 3f361dd7ba..923ed0a17d 100644 --- a/test/Extensions/ServiceBus.Tests/Streaming/EHProgrammaticSubscribeTests.cs +++ b/test/Extensions/ServiceBus.Tests/Streaming/EHProgrammaticSubscribeTests.cs @@ -8,7 +8,7 @@ namespace ServiceBus.Tests.Streaming { [TestCategory("EventHub"), TestCategory("Streaming"), TestCategory("Functional")] - public class EHProgrammaticSubscribeTest : ProgrammaticSubcribeTestsRunner, IClassFixture + public class EHProgrammaticSubscribeTest : ProgrammaticSubscribeTestsRunner, IClassFixture { private const string EHPath = "ehorleanstest4"; private const string EHPath2 = "ehorleanstest3"; diff --git a/test/Extensions/TesterAzureUtils/Streaming/AQProgrammaticSubscribeTest.cs b/test/Extensions/TesterAzureUtils/Streaming/AQProgrammaticSubscribeTest.cs index 3ed545ae6f..1649fa6bd0 100644 --- a/test/Extensions/TesterAzureUtils/Streaming/AQProgrammaticSubscribeTest.cs +++ b/test/Extensions/TesterAzureUtils/Streaming/AQProgrammaticSubscribeTest.cs @@ -12,7 +12,7 @@ namespace Tester.AzureUtils.Streaming { [TestCategory("BVT"), TestCategory("Streaming"), TestCategory("AQStreaming")] - public class AQProgrammaticSubscribeTest : ProgrammaticSubcribeTestsRunner, IClassFixture + public class AQProgrammaticSubscribeTest : ProgrammaticSubscribeTestsRunner, IClassFixture { private const int queueCount = 8; public class Fixture : BaseAzureTestClusterFixture diff --git a/test/Grains/TestGrainInterfaces/ITimerGrain.cs b/test/Grains/TestGrainInterfaces/ITimerGrain.cs index 5475c6e901..6a81b2d412 100644 --- a/test/Grains/TestGrainInterfaces/ITimerGrain.cs +++ b/test/Grains/TestGrainInterfaces/ITimerGrain.cs @@ -32,5 +32,18 @@ public interface ITimerRequestGrain : IGrainWithIntegerKey Task StartStuckTimer(TimeSpan dueTime); Task GetRuntimeInstanceId(); + Task TestAllTimerOverloads(); + Task PollCompletedTimers(); + Task TestCompletedTimerResults(); + } + + public interface INonReentrantTimerCallGrain : IGrainWithIntegerKey + { + Task GetTickCount(); + Task GetException(); + + Task StartTimer(string name, TimeSpan delay, bool keepAlive = true); + Task StopTimer(string name); + Task ExternalTick(string name); } } diff --git a/test/Grains/TestGrains/GenericGrains.cs b/test/Grains/TestGrains/GenericGrains.cs index 04a7a51999..ac7196f59f 100644 --- a/test/Grains/TestGrains/GenericGrains.cs +++ b/test/Grains/TestGrains/GenericGrains.cs @@ -1,5 +1,6 @@ using System.Globalization; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.ObjectPool; using Orleans.Concurrency; using Orleans.Providers; using Orleans.Runtime; @@ -576,16 +577,15 @@ public Task PingSelfThroughOther(IGenericPingSelf target, T t) public Task ScheduleDelayedPing(IGenericPingSelf target, T t, TimeSpan delay) { - _timerRegistry.RegisterTimer( + _timerRegistry.RegisterGrainTimer( GrainContext, - o => + (_, cancellationToken) => { this.logger.LogDebug("***Timer fired for pinging {0}***", target.GetPrimaryKey()); return target.Ping(t); }, null, - delay, - TimeSpan.FromMilliseconds(-1)); + new() { DueTime = delay, Period = Timeout.InfiniteTimeSpan }); return Task.CompletedTask; } diff --git a/test/Grains/TestGrains/LivenessTestGrain.cs b/test/Grains/TestGrains/LivenessTestGrain.cs index f87503f460..fe4f445ed4 100644 --- a/test/Grains/TestGrains/LivenessTestGrain.cs +++ b/test/Grains/TestGrains/LivenessTestGrain.cs @@ -47,12 +47,12 @@ public Task SetLabel(string label) public Task StartTimer() { logger.LogInformation("StartTimer."); - base.RegisterTimer(TimerTick, null, TimeSpan.Zero, TimeSpan.FromSeconds(10)); + this.RegisterGrainTimer(TimerTick, TimeSpan.Zero, TimeSpan.FromSeconds(10)); return Task.CompletedTask; } - private Task TimerTick(object data) + private Task TimerTick() { logger.LogInformation("TimerTick."); return Task.CompletedTask; diff --git a/test/Grains/TestGrains/ProgrammaticSubscribe/TypedProducerGrain.cs b/test/Grains/TestGrains/ProgrammaticSubscribe/TypedProducerGrain.cs index 90897efaab..03226eef20 100644 --- a/test/Grains/TestGrains/ProgrammaticSubscribe/TypedProducerGrain.cs +++ b/test/Grains/TestGrains/ProgrammaticSubscribe/TypedProducerGrain.cs @@ -38,7 +38,7 @@ public Task StartPeriodicProducing(TimeSpan? firePeriod = null) { logger.LogInformation("StartPeriodicProducing"); var period = (firePeriod == null)? defaultFirePeriod : firePeriod; - producerTimer = base.RegisterTimer(TimerCallback, null, TimeSpan.Zero, period.Value); + producerTimer = this.RegisterGrainTimer(TimerCallback, TimeSpan.Zero, period.Value); return Task.CompletedTask; } @@ -72,7 +72,7 @@ public Task Produce() return Fire(); } - private Task TimerCallback(object state) + private Task TimerCallback() { return producerTimer != null ? Fire() : Task.CompletedTask; } diff --git a/test/Grains/TestGrains/SampleStreamingGrain.cs b/test/Grains/TestGrains/SampleStreamingGrain.cs index 6d29d6bd36..ce36258dbe 100644 --- a/test/Grains/TestGrains/SampleStreamingGrain.cs +++ b/test/Grains/TestGrains/SampleStreamingGrain.cs @@ -1,6 +1,6 @@ using System.Runtime.CompilerServices; using Microsoft.Extensions.Logging; -using Orleans.Runtime; +using Orleans; using Orleans.Streams; using UnitTests.GrainInterfaces; @@ -67,7 +67,7 @@ public Task BecomeProducer(Guid streamId, string streamNamespace, string provide public Task StartPeriodicProducing() { logger.LogInformation("StartPeriodicProducing"); - producerTimer = base.RegisterTimer(TimerCallback, null, TimeSpan.Zero, TimeSpan.FromMilliseconds(10)); + producerTimer = this.RegisterGrainTimer(TimerCallback, TimeSpan.Zero, TimeSpan.FromMilliseconds(10)); return Task.CompletedTask; } @@ -96,7 +96,7 @@ public Task Produce() return Fire(); } - private Task TimerCallback(object state) + private Task TimerCallback() { return producerTimer != null? Fire(): Task.CompletedTask; } diff --git a/test/Grains/TestGrains/StatelessWorkerGrain.cs b/test/Grains/TestGrains/StatelessWorkerGrain.cs index 5fa4eefcab..9ca0e34a4f 100644 --- a/test/Grains/TestGrains/StatelessWorkerGrain.cs +++ b/test/Grains/TestGrains/StatelessWorkerGrain.cs @@ -1,6 +1,5 @@ using Microsoft.Extensions.Logging; using Orleans.Concurrency; -using Orleans.Runtime; using UnitTests.GrainInterfaces; @@ -41,7 +40,7 @@ public Task LongCall() } DateTime start = DateTime.UtcNow; TaskCompletionSource resolver = new TaskCompletionSource(); - RegisterTimer(TimerCallback, resolver, TimeSpan.FromSeconds(2), TimeSpan.FromMilliseconds(-1)); + this.RegisterGrainTimer(TimerCallback, resolver, new() { DueTime = TimeSpan.FromSeconds(2), Period = Timeout.InfiniteTimeSpan, Interleave = true }); return resolver.Task.ContinueWith( (_) => { @@ -57,13 +56,12 @@ public Task LongCall() }); } - private static Task TimerCallback(object state) + private static Task TimerCallback(TaskCompletionSource state, CancellationToken cancellationToken) { - ((TaskCompletionSource)state).SetResult(true); + state.SetResult(true); return Task.CompletedTask; } - public Task>>> GetCallStats() { Thread.Sleep(200); diff --git a/test/Grains/TestGrains/StuckGrain.cs b/test/Grains/TestGrains/StuckGrain.cs index b667c53ef2..b49ffae0a2 100644 --- a/test/Grains/TestGrains/StuckGrain.cs +++ b/test/Grains/TestGrains/StuckGrain.cs @@ -1,6 +1,5 @@ using System.Collections.Concurrent; using Microsoft.Extensions.Logging; -using Orleans.Runtime; using UnitTests.GrainInterfaces; diff --git a/test/Grains/TestInternalGrains/CollectionTestGrain.cs b/test/Grains/TestInternalGrains/CollectionTestGrain.cs index f1c408dd08..f3cef97bef 100644 --- a/test/Grains/TestInternalGrains/CollectionTestGrain.cs +++ b/test/Grains/TestInternalGrains/CollectionTestGrain.cs @@ -89,15 +89,15 @@ public Task GetGrainReference() Logger().LogInformation("GetGrainReference."); return Task.FromResult(this.AsReference()); } + public Task StartTimer(TimeSpan timerPeriod, TimeSpan delayPeriod) { - RegisterTimer(TimerCallback, delayPeriod, TimeSpan.Zero, timerPeriod); + this.RegisterGrainTimer(TimerCallback, delayPeriod, TimeSpan.Zero, timerPeriod); return Task.CompletedTask; } - private async Task TimerCallback(object state) + private async Task TimerCallback(TimeSpan delayPeriod, CancellationToken cancellationToken) { - TimeSpan delayPeriod = (TimeSpan)state; staticCounter++; counter++; int tmpCounter = counter; diff --git a/test/Grains/TestInternalGrains/StreamingGrain.cs b/test/Grains/TestInternalGrains/StreamingGrain.cs index 2f3c940438..1d0d71ed47 100644 --- a/test/Grains/TestInternalGrains/StreamingGrain.cs +++ b/test/Grains/TestInternalGrains/StreamingGrain.cs @@ -511,7 +511,7 @@ public virtual async Task ProducePeriodicSeries(int count) await Task.WhenAll(_producers.Select(p => p.ProducePeriodicSeries(timerCallback => { - return RegisterTimer(timerCallback, null, TimeSpan.Zero, TimeSpan.FromMilliseconds(10)); + return this.RegisterGrainTimer(timerCallback, null, TimeSpan.Zero, TimeSpan.FromMilliseconds(10)); },count)).ToArray()); } @@ -815,7 +815,7 @@ public Task ProducePeriodicSeries(int count) { return _producer.ProducePeriodicSeries(timerCallback => { - return RegisterTimer(timerCallback, null, TimeSpan.Zero, TimeSpan.FromMilliseconds(10)); + return this.RegisterGrainTimer(timerCallback, null, TimeSpan.Zero, TimeSpan.FromMilliseconds(10)); }, count); } diff --git a/test/Grains/TestInternalGrains/TestGrain.cs b/test/Grains/TestInternalGrains/TestGrain.cs index 0fa918fcea..401b585a37 100644 --- a/test/Grains/TestInternalGrains/TestGrain.cs +++ b/test/Grains/TestInternalGrains/TestGrain.cs @@ -61,12 +61,13 @@ public Task SetLabel(string label) public Task StartTimer() { logger.LogInformation("StartTimer."); - timer = base.RegisterTimer(TimerTick, null, TimeSpan.Zero, TimeSpan.FromSeconds(10)); + timer = RegisterGrainTimer(TimerTick, dueTime: TimeSpan.Zero, period: TimeSpan.FromSeconds(10)); return Task.CompletedTask; } + private Task Ticker(object obj) => Task.CompletedTask; - private Task TimerTick(object data) + private Task TimerTick(CancellationToken cancellationToken) { logger.LogInformation("TimerTick."); return Task.CompletedTask; diff --git a/test/Grains/TestInternalGrains/TimerGrain.cs b/test/Grains/TestInternalGrains/TimerGrain.cs index 730f6e608b..3c799af27e 100644 --- a/test/Grains/TestInternalGrains/TimerGrain.cs +++ b/test/Grains/TestInternalGrains/TimerGrain.cs @@ -3,6 +3,7 @@ using Orleans.Runtime.Scheduler; using UnitTests.GrainInterfaces; using UnitTests.Grains; +using Xunit; namespace UnitTestGrains @@ -28,7 +29,7 @@ public override Task OnActivateAsync(CancellationToken cancellationToken) { ThrowIfDeactivating(); context = RuntimeContext.Current; - defaultTimer = this.RegisterTimer(Tick, DefaultTimerName, period, period); + defaultTimer = this.RegisterGrainTimer(Tick, DefaultTimerName, period, period); allTimers = new Dictionary(); return Task.CompletedTask; } @@ -49,7 +50,7 @@ private Task Tick(object data) RuntimeContext.Current); // make sure we run in the right activation context. - if(!Equals(context, RuntimeContext.Current)) + if (!Equals(context, RuntimeContext.Current)) logger.LogError((int)ErrorCode.Runtime_Error_100146, "Grain not running in the right activation context"); string name = (string)data; @@ -62,7 +63,7 @@ private Task Tick(object data) { timer = allTimers[(string)data]; } - if(timer == null) + if (timer == null) logger.LogError((int)ErrorCode.Runtime_Error_100146, "Timer is null"); if (timer != null && counter > 10000) { @@ -95,7 +96,7 @@ public Task SetCounter(int value) public Task StartTimer(string timerName) { ThrowIfDeactivating(); - IDisposable timer = this.RegisterTimer(Tick, timerName, TimeSpan.Zero, period); + IDisposable timer = this.RegisterGrainTimer(Tick, timerName, TimeSpan.Zero, period); allTimers.Add(timerName, timer); return Task.CompletedTask; } @@ -158,7 +159,7 @@ public Task StartTimer(string name, TimeSpan delay) { logger.LogInformation("StartTimer Name={Name} Delay={Delay}", name, delay); if (timer is not null) throw new InvalidOperationException("Expected timer to be null"); - this.timer = (IGrainTimer)base.RegisterTimer(TimerTick, name, delay, Constants.INFINITE_TIMESPAN); // One shot timer + this.timer = this.RegisterGrainTimer(TimerTick, name, new(delay, Timeout.InfiniteTimeSpan)); // One shot timer this.timerName = name; return Task.CompletedTask; @@ -169,7 +170,7 @@ public Task StartTimer(string name, TimeSpan delay, string operationType) logger.LogInformation("StartTimer Name={Name} Delay={Delay}", name, delay); if (timer is not null) throw new InvalidOperationException("Expected timer to be null"); var state = Tuple.Create(operationType, name); - this.timer = (IGrainTimer)base.RegisterTimer(TimerTickAdvanced, state, delay, Constants.INFINITE_TIMESPAN); // One shot timer + this.timer = this.RegisterGrainTimer(TimerTickAdvanced, state, delay, Timeout.InfiniteTimeSpan); // One shot timer this.timerName = name; return Task.CompletedTask; @@ -179,7 +180,7 @@ public Task RestartTimer(string name, TimeSpan delay) { logger.LogInformation("RestartTimer Name={Name} Delay={Delay}", name, delay); this.timerName = name; - timer.Change(delay, Constants.INFINITE_TIMESPAN); + timer.Change(delay, Timeout.InfiniteTimeSpan); return Task.CompletedTask; } @@ -291,7 +292,7 @@ private async Task ProcessTimerTick(object data) private void CheckRuntimeContext(string what) { - if (RuntimeContext.Current == null + if (RuntimeContext.Current == null || !RuntimeContext.Current.Equals(context)) { throw new InvalidOperationException( @@ -323,9 +324,170 @@ private void LogStatus(string what) } } + public class NonReentrantTimerCallGrain : Grain, INonReentrantTimerCallGrain + { + private readonly Dictionary _timers = []; + private int _tickCount; + private Exception _tickException; + private IGrainContext _context; + private TaskScheduler _activationTaskScheduler; + private Guid _tickId; + + private readonly ILogger _logger; + + public NonReentrantTimerCallGrain(ILoggerFactory loggerFactory) + { + _logger = loggerFactory.CreateLogger($"{GetType().Name}-{IdentityString}"); + } + + public Task GetTickCount() => Task.FromResult(_tickCount); + public Task GetException() => Task.FromResult(_tickException); + + public async Task ExternalTick(string name) + { + await ProcessTimerTick(name, CancellationToken.None); + } + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + _context = RuntimeContext.Current; + _activationTaskScheduler = TaskScheduler.Current; + return Task.CompletedTask; + } + + public Task StartTimer(string name, TimeSpan delay, bool keepAlive) + { + _logger.LogInformation("StartTimer Name={Name} Delay={Delay}", name, delay); + if (_timers.TryGetValue(name, out var timer)) + { + // Make the timer fire again after the specified delay. + timer.Change(delay, Timeout.InfiniteTimeSpan); + } + else + { + _timers[name] = RegisterGrainTimer(TimerTick, name, new() { DueTime = delay, Period = Timeout.InfiniteTimeSpan, KeepAlive = keepAlive }); // One shot timer + } + + return Task.CompletedTask; + } + + public Task StopTimer(string name) + { + _logger.LogInformation("StopTimer Name={Name}", name); + + if (!_timers.Remove(name, out var timer)) + { + throw new ArgumentException($"Could not find a timer with name {name}."); + } + + timer.Dispose(); + return Task.CompletedTask; + } + + private async Task TimerTick(object data, CancellationToken cancellationToken) + { + try + { + await ProcessTimerTick(data, cancellationToken); + } + catch (Exception exc) + { + _tickException = exc; + throw; + } + } + + private async Task ProcessTimerTick(object data, CancellationToken cancellationToken) + { + var timerName = (string)data; + string step = "TimerTick"; + CheckReentrancy(step, Guid.Empty); + var expectedTickId = _tickId = Guid.NewGuid(); + LogStatus(step, timerName); + CheckRuntimeContext(step); + CheckReentrancy(step, expectedTickId); + + ISimpleGrain grain = GrainFactory.GetGrain(0, SimpleGrain.SimpleGrainNamePrefix); + + LogStatus("Before grain call #1", timerName); + await grain.SetA(_tickCount); + step = "After grain call #1"; + LogStatus(step, timerName); + CheckRuntimeContext(step); + CheckReentrancy(step, expectedTickId); + + LogStatus("Before Delay", timerName); + await Task.Delay(TimeSpan.FromSeconds(1)); + step = "After Delay"; + LogStatus(step, timerName); + CheckRuntimeContext(step); + CheckReentrancy(step, expectedTickId); + + LogStatus("Before grain call #2", timerName); + await grain.SetB(_tickCount); + step = "After grain call #2"; + LogStatus(step, timerName); + CheckRuntimeContext(step); + CheckReentrancy(step, expectedTickId); + + LogStatus("Before grain call #3", timerName); + int res = await grain.GetAxB(); + step = "After grain call #3 - Result = " + res; + LogStatus(step, timerName); + CheckRuntimeContext(step); + CheckReentrancy(step, expectedTickId); + + _tickCount++; + _tickId = Guid.Empty; + } + + private void CheckRuntimeContext(string what) + { + if (RuntimeContext.Current == null + || !RuntimeContext.Current.Equals(_context)) + { + throw new InvalidOperationException( + string.Format("{0} in timer callback with unexpected activation context: Expected={1} Actual={2}", + what, _context, RuntimeContext.Current)); + } + if (TaskScheduler.Current.Equals(_activationTaskScheduler) && TaskScheduler.Current is ActivationTaskScheduler) + { + // Everything is as expected + } + else + { + throw new InvalidOperationException( + string.Format("{0} in timer callback with unexpected TaskScheduler.Current context: Expected={1} Actual={2}", + what, _activationTaskScheduler, TaskScheduler.Current)); + } + } + + private void CheckReentrancy(string what, Guid expected) + { + if (_tickId != expected) + { + throw new InvalidOperationException( + $"{what} in timer callback with unexpected interleaving: Expected={expected} Actual={_tickId}"); + } + } + + private void LogStatus(string what, string timerName) + { + _logger.LogInformation( + "{TimerName} Tick # {TickCount} - {Step} - RuntimeContext.Current={RuntimeContext} TaskScheduler.Current={TaskScheduler} CurrentWorkerThread={Thread}", + timerName, + _tickCount, + what, + RuntimeContext.Current, + TaskScheduler.Current, + Thread.CurrentThread.Name); + } + } + public class TimerRequestGrain : Grain, ITimerRequestGrain { private TaskCompletionSource completionSource; + private List> _allTimerCallsTasks; public Task GetRuntimeInstanceId() { @@ -335,26 +497,134 @@ public Task GetRuntimeInstanceId() public async Task StartAndWaitTimerTick(TimeSpan dueTime) { this.completionSource = new TaskCompletionSource(); - var timer = this.RegisterTimer(TimerTick, null, dueTime, TimeSpan.FromMilliseconds(-1)); + using var timer = this.RegisterGrainTimer(TimerTick, new() { DueTime = dueTime, Period = Timeout.InfiniteTimeSpan, Interleave = true }); await this.completionSource.Task; } public Task StartStuckTimer(TimeSpan dueTime) { this.completionSource = new TaskCompletionSource(); - var timer = this.RegisterTimer(StuckTimerTick, null, dueTime, TimeSpan.FromSeconds(1)); + var timer = this.RegisterGrainTimer(StuckTimerTick, new() { DueTime = dueTime, Period = TimeSpan.FromSeconds(1), Interleave = true }); return Task.CompletedTask; } - private Task TimerTick(object state) + private Task TimerTick() { this.completionSource.SetResult(1); return Task.CompletedTask; } - private async Task StuckTimerTick(object state) + private async Task StuckTimerTick(CancellationToken cancellationToken) { await completionSource.Task; } + + public Task TestAllTimerOverloads() + { + var tasks = new List>(); + var timers = new List(); + + // protected IGrainTimer RegisterGrainTimer(Func callback, GrainTimerCreationOptions options) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer(() => + { + tasks[0].SetResult(("NONE", CancellationToken.None)); + return Task.CompletedTask; + }, new GrainTimerCreationOptions(TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10)) { Interleave = true })); + + // protected IGrainTimer RegisterGrainTimer(Func callback, GrainTimerCreationOptions options) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer(() => + { + tasks[1].SetResult(("NONE", CancellationToken.None)); + return Task.CompletedTask; + }, TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10))); + + // protected IGrainTimer RegisterGrainTimer(Func callback, TState state, GrainTimerCreationOptions options) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer(state => + { + tasks[2].SetResult((state, CancellationToken.None)); + return Task.CompletedTask; + }, + "STATE", + new GrainTimerCreationOptions(TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10)) { Interleave = true })); + + // protected IGrainTimer RegisterGrainTimer(Func callback, TState state, TimeSpan dueTime, TimeSpan period) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer(state => + { + tasks[3].SetResult((state, CancellationToken.None)); + return Task.CompletedTask; + }, + "STATE", + TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10))); + + // With CancellationToken + // protected IGrainTimer RegisterGrainTimer(Func callback, GrainTimerCreationOptions options) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer(ct => + { + tasks[4].SetResult(("NONE", ct)); + return Task.CompletedTask; + }, new GrainTimerCreationOptions(TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10)) { Interleave = true })); + + // protected IGrainTimer RegisterGrainTimer(Func callback, TimeSpan dueTime, TimeSpan period) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer(ct => + { + tasks[5].SetResult(("NONE", ct)); + return Task.CompletedTask; + }, TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10))); + + // protected internal IGrainTimer RegisterGrainTimer(Func callback, TState state, GrainTimerCreationOptions options) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer((state, ct) => + { + tasks[6].SetResult((state, ct)); + return Task.CompletedTask; + }, + "STATE", + new GrainTimerCreationOptions(TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10)) { Interleave = true })); + + // protected IGrainTimer RegisterGrainTimer(Func callback, TState state, TimeSpan dueTime, TimeSpan period) + tasks.Add(new()); + timers.Add(this.RegisterGrainTimer((state, ct) => + { + tasks[7].SetResult((state, ct)); + return Task.CompletedTask; + }, + "STATE", + TimeSpan.FromMilliseconds(25), TimeSpan.FromSeconds(10))); + _allTimerCallsTasks = tasks; + return Task.FromResult(_allTimerCallsTasks.Count); + } + + public Task PollCompletedTimers() => Task.FromResult(_allTimerCallsTasks.Count(c => c.Task.IsCompleted)); + public async Task TestCompletedTimerResults() + { + var countWithState = 0; + var countWithCancellation = 0; + + foreach (var task in _allTimerCallsTasks.Select(t => t.Task)) + { + var (state, ct) = await task; + var stateString = Assert.IsType(state); + var hasState = string.Equals("STATE", stateString, StringComparison.Ordinal); + if (hasState) + { + countWithState++; + } + + Assert.True(hasState || string.Equals("NONE", stateString, StringComparison.Ordinal)); + if (ct.CanBeCanceled) + { + countWithCancellation++; + } + } + + Assert.Equal(4, countWithState); + Assert.Equal(4, countWithCancellation); + } } } diff --git a/test/NonSilo.Tests/Async_AsyncExecutorWithRetriesTests.cs b/test/NonSilo.Tests/Async_AsyncExecutorWithRetriesTests.cs index 05dc2fa179..663a7de3cf 100644 --- a/test/NonSilo.Tests/Async_AsyncExecutorWithRetriesTests.cs +++ b/test/NonSilo.Tests/Async_AsyncExecutorWithRetriesTests.cs @@ -75,7 +75,7 @@ public async Task Async_AsyncExecutorWithRetriesTest_2() int maxRetries = 10; int expectedRetries = countLimit; - Task promise = AsyncExecutorWithRetries.ExecuteWithRetries(myFunc, maxRetries, maxRetries, successFilter, null, Constants.INFINITE_TIMESPAN); + Task promise = AsyncExecutorWithRetries.ExecuteWithRetries(myFunc, maxRetries, maxRetries, successFilter, null, Timeout.InfiniteTimeSpan); int value = await promise; this.output.WriteLine("Value={0} Counter={1} ExpectedRetries={2}", value, counter, expectedRetries); Assert.Equal(expectedRetries, value); // "Returned value" diff --git a/test/NonSilo.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs b/test/NonSilo.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs index 2e35235a6a..c9a298c839 100644 --- a/test/NonSilo.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs +++ b/test/NonSilo.Tests/SchedulerTests/OrleansTaskSchedulerBasicTests.cs @@ -49,8 +49,8 @@ public static UnitTestSchedulingContext Create(ILoggerFactory loggerFactory) object IGrainContext.GrainInstance => throw new NotImplementedException(); - public void Activate(Dictionary requestContext, CancellationToken? cancellationToken = default) => throw new NotImplementedException(); - public void Deactivate(DeactivationReason deactivationReason, CancellationToken? cancellationToken = default) { } + public void Activate(Dictionary requestContext, CancellationToken cancellationToken) => throw new NotImplementedException(); + public void Deactivate(DeactivationReason deactivationReason, CancellationToken cancellationToken) { } public Task Deactivated => Task.CompletedTask; public void Dispose() => (Scheduler as IDisposable)?.Dispose(); public TComponent GetComponent() where TComponent : class => throw new NotImplementedException(); @@ -61,7 +61,7 @@ public static UnitTestSchedulingContext Create(ILoggerFactory loggerFactory) bool IEquatable.Equals(IGrainContext other) => ReferenceEquals(this, other); void IGrainContext.Rehydrate(IRehydrationContext context) => throw new NotImplementedException(); - void IGrainContext.Migrate(Dictionary requestContext, CancellationToken? cancellationToken) => throw new NotImplementedException(); + void IGrainContext.Migrate(Dictionary requestContext, CancellationToken cancellationToken) => throw new NotImplementedException(); } [TestCategory("BVT"), TestCategory("Scheduler")] diff --git a/test/Tester/Forwarding/ShutdownSiloTests.cs b/test/Tester/Forwarding/ShutdownSiloTests.cs index 29b94e31d5..0b46212ccb 100644 --- a/test/Tester/Forwarding/ShutdownSiloTests.cs +++ b/test/Tester/Forwarding/ShutdownSiloTests.cs @@ -4,7 +4,6 @@ using Xunit; using Orleans.Configuration; using System.Diagnostics; -using Orleans.Runtime; using Microsoft.Extensions.DependencyInjection; namespace Tester.Forwarding diff --git a/test/Tester/GrainCallFilterTests.cs b/test/Tester/GrainCallFilterTests.cs index 953ddb9e10..02ba146374 100644 --- a/test/Tester/GrainCallFilterTests.cs +++ b/test/Tester/GrainCallFilterTests.cs @@ -263,13 +263,14 @@ public async Task GrainCallFilter_Incoming_Stream_Test() // The intercepted grain should double the value passed to the stream. const int testValue = 43; await stream.OnNextAsync(testValue); - var cts = new CancellationTokenSource(1000); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); int actual = 0; while (!cts.IsCancellationRequested) { actual = await grain.GetLastStreamValue(); if (actual != 0) break; } + Assert.Equal(testValue * 2, actual); } diff --git a/test/Tester/StreamingTests/MemoryProgrammaticSubcribeTests.cs b/test/Tester/StreamingTests/MemoryProgrammaticSubcribeTests.cs index 44d5dbfe3a..66974fd01e 100644 --- a/test/Tester/StreamingTests/MemoryProgrammaticSubcribeTests.cs +++ b/test/Tester/StreamingTests/MemoryProgrammaticSubcribeTests.cs @@ -8,7 +8,7 @@ namespace UnitTests.StreamingTests { [TestCategory("BVT"), TestCategory("Streaming")] - public class MemoryProgrammaticSubcribeTests : ProgrammaticSubcribeTestsRunner, IClassFixture + public class MemoryProgrammaticSubcribeTests : ProgrammaticSubscribeTestsRunner, IClassFixture { public class Fixture : BaseTestClusterFixture { diff --git a/test/Tester/StreamingTests/ProgrammaticSubscribeTests/ProgrammaticSubcribeTestsRunner.cs b/test/Tester/StreamingTests/ProgrammaticSubscribeTests/ProgrammaticSubscribeTestsRunner.cs similarity index 99% rename from test/Tester/StreamingTests/ProgrammaticSubscribeTests/ProgrammaticSubcribeTestsRunner.cs rename to test/Tester/StreamingTests/ProgrammaticSubscribeTests/ProgrammaticSubscribeTestsRunner.cs index 22f636737b..6bd8fc5353 100644 --- a/test/Tester/StreamingTests/ProgrammaticSubscribeTests/ProgrammaticSubcribeTestsRunner.cs +++ b/test/Tester/StreamingTests/ProgrammaticSubscribeTests/ProgrammaticSubscribeTestsRunner.cs @@ -11,12 +11,12 @@ namespace Tester.StreamingTests { - public abstract class ProgrammaticSubcribeTestsRunner + public abstract class ProgrammaticSubscribeTestsRunner { private readonly BaseTestClusterFixture fixture; public const string StreamProviderName = "StreamProvider1"; public const string StreamProviderName2 = "StreamProvider2"; - public ProgrammaticSubcribeTestsRunner(BaseTestClusterFixture fixture) + public ProgrammaticSubscribeTestsRunner(BaseTestClusterFixture fixture) { this.fixture = fixture; } diff --git a/test/Tester/StreamingTests/ProgrammaticSubscribeTests/SubscriptionObserverWithImplicitSubscribingTestRunner.cs b/test/Tester/StreamingTests/ProgrammaticSubscribeTests/SubscriptionObserverWithImplicitSubscribingTestRunner.cs index eb635c1c48..88fdf485dc 100644 --- a/test/Tester/StreamingTests/ProgrammaticSubscribeTests/SubscriptionObserverWithImplicitSubscribingTestRunner.cs +++ b/test/Tester/StreamingTests/ProgrammaticSubscribeTests/SubscriptionObserverWithImplicitSubscribingTestRunner.cs @@ -40,7 +40,7 @@ public async Task StreamingTests_Consumer_Producer_Subscribe() var implicitConsumer = this.fixture.HostedCluster.GrainFactory.GetGrain(streamId.Guid); - await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.CheckCounters(new List { producer }, + await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubscribeTestsRunner.CheckCounters(new List { producer }, implicitConsumer, lastTry, this.fixture.Logger), _timeout); //clean up test @@ -70,7 +70,7 @@ public async Task StreamingTests_Consumer_Producer_SubscribeToTwoStream_MessageW } var implicitConsumer = this.fixture.HostedCluster.GrainFactory.GetGrain(streamId.Guid); - await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.CheckCounters(new List { producer, producer2 }, + await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubscribeTestsRunner.CheckCounters(new List { producer, producer2 }, implicitConsumer, lastTry, this.fixture.Logger), _timeout); //clean up test @@ -102,7 +102,7 @@ public async Task StreamingTests_Consumer_Producer_SubscribeToStreamsHandledByDi var implicitConsumer = this.fixture.HostedCluster.GrainFactory.GetGrain(streamId.Guid); - await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubcribeTestsRunner.CheckCounters(new List { producer, producer2 }, + await TestingUtils.WaitUntilAsync(lastTry => ProgrammaticSubscribeTestsRunner.CheckCounters(new List { producer, producer2 }, implicitConsumer, lastTry, this.fixture.Logger), _timeout); //clean up test diff --git a/test/TesterInternal/ActivationsLifeCycleTests/ActivationCollectorTests.cs b/test/TesterInternal/ActivationsLifeCycleTests/ActivationCollectorTests.cs index 4da0903d63..6144a79055 100644 --- a/test/TesterInternal/ActivationsLifeCycleTests/ActivationCollectorTests.cs +++ b/test/TesterInternal/ActivationsLifeCycleTests/ActivationCollectorTests.cs @@ -7,6 +7,7 @@ using Orleans.TestingHost; using Tester; using TestExtensions; +using UnitTestGrains; using UnitTests.GrainInterfaces; using UnitTests.Grains; using Xunit; @@ -574,5 +575,25 @@ public async Task ActivationCollectorShouldCollectByCollectionSpecificAgeLimitFo int activationsNotCollected = await TestUtils.GetActivationCount(this.testCluster.GrainFactory, fullGrainTypeName); Assert.Equal(0, activationsNotCollected); } + + [Fact, TestCategory("SlowBVT"), TestCategory("Timers")] + public async Task NonReentrantGrainTimer_NoKeepAlive_Test() + { + await Initialize(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(1)); + + const string testName = "NonReentrantGrainTimer_NoKeepAlive_Test"; + + var grain = this.testCluster.GrainFactory.GetGrain(GetRandomGrainId()); + + // Schedule a timer to fire at the 30s mark which will not extend the grain's lifetime. + await grain.StartTimer(testName, TimeSpan.FromSeconds(4), keepAlive: false); + await Task.Delay(TimeSpan.FromSeconds(7)); + + var tickCount = await grain.GetTickCount(); + + // The grain should have been deactivated. + Assert.Equal(0, tickCount); + } + } } diff --git a/test/TesterInternal/ActivationsLifeCycleTests/DeactivateOnIdleTests.cs b/test/TesterInternal/ActivationsLifeCycleTests/DeactivateOnIdleTests.cs index 180f192758..6aa1941b0f 100644 --- a/test/TesterInternal/ActivationsLifeCycleTests/DeactivateOnIdleTests.cs +++ b/test/TesterInternal/ActivationsLifeCycleTests/DeactivateOnIdleTests.cs @@ -13,8 +13,6 @@ namespace UnitTests.ActivationsLifeCycleTests { - - [TestCategory("ActivationCollector")] public class DeactivateOnIdleTests : OrleansTestingBase, IDisposable { From 51fdad8a9370b060cd99527e66ea5104aa56087f Mon Sep 17 00:00:00 2001 From: Ledjon Behluli Date: Sun, 19 May 2024 10:33:12 -0700 Subject: [PATCH 02/28] Active Rebalancing --- distributed-tests.yml | 51 +- src/Orleans.Core.Abstractions/IDs/GrainId.cs | 2 +- .../Manifest/GrainProperties.cs | 5 + .../Placement/PlacementAttribute.cs | 12 + .../Configuration/Options/MessagingOptions.cs | 2 +- .../Core/DefaultClientServices.cs | 2 + .../Metrics/ApplicationRequestInstruments.cs | 21 +- .../Diagnostics/Metrics/CatalogInstruments.cs | 8 +- .../Diagnostics/Metrics/InstrumentNames.cs | 259 +++--- .../Metrics/MessagingInstruments.cs | 40 +- src/Orleans.Core/Messaging/Message.cs | 3 +- .../Networking/ClientOutboundConnection.cs | 5 +- .../ClientOutboundConnectionFactory.cs | 1 + src/Orleans.Core/Networking/Connection.cs | 5 +- .../Networking/ConnectionShared.cs | 31 +- .../IActivationRebalancerSystemTarget.cs | 190 +++++ .../Rebalancing/IImbalanceToleranceRule.cs | 13 + .../Rebalancing/IMessageStatisticsSink.cs | 13 + src/Orleans.Core/Runtime/Constants.cs | 1 + .../Activation/IGrainContextActivator.cs | 57 +- .../Catalog/ActivationCollector.cs | 2 +- src/Orleans.Runtime/Catalog/ActivationData.cs | 10 +- src/Orleans.Runtime/Catalog/Catalog.cs | 10 +- .../Catalog/GrainTypeSharedContext.cs | 333 ++++---- .../Options/ActiveRebalancingOptions.cs | 150 ++++ .../Options/SiloMessagingOptions.cs | 2 +- .../Hosting/ActiveRebalancingExtensions.cs | 51 ++ .../Hosting/DefaultSiloServices.cs | 2 + .../MembershipService/ISiloStatusOracle.cs | 6 + .../MembershipService/SiloStatusOracle.cs | 32 +- .../Messaging/MessageCenter.cs | 18 +- .../Networking/GatewayInboundConnection.cs | 5 +- .../Networking/SiloConnection.cs | 5 +- .../Placement/PlacementService.cs | 2 +- .../Rebalancing/ActivationRebalancer.Log.cs | 40 + .../ActivationRebalancer.MessageSink.cs | 90 +++ .../Rebalancing/ActivationRebalancer.cs | 759 ++++++++++++++++++ .../Rebalancing/DefaultImbalanceRule.cs | 56 ++ .../Rebalancing/FrequentItemCollection.cs | 397 +++++++++ .../Placement/Rebalancing/MaxHeap.cs | 281 +++++++ .../Rebalancing/RebalancingMessageFilter.cs | 129 +++ .../Placement/Rebalancing/WeightedEdge.cs | 22 + .../Properties/AssemblyInfo.cs | 1 + .../Utilities/StripedMpscBuffer.cs | 425 ++++++++++ test/Benchmarks/Ping/FanoutBenchmark.cs | 150 ++++ test/Benchmarks/Ping/PingBenchmark.cs | 12 + test/Benchmarks/Ping/TreeGrain.cs | 41 + test/Benchmarks/Program.cs | 10 + .../Benchmarks/Properties/launchSettings.json | 2 +- test/Benchmarks/TopK/TopKBenchmark.cs | 391 +++++++++ .../ConcurrentLoadGenerator.cs | 6 + .../LoadGeneratorScenarioRunner.cs | 16 +- .../LoadGeneratorScenarios.cs | 9 + .../DistributedTests.Client/Program.cs | 1 + .../GrainInterfaces/IPingGrain.cs | 10 +- .../GrainInterfaces/ITreeGrain.cs | 7 + .../DistributedTests.Grains/PingGrain.cs | 9 +- .../DistributedTests.Grains/TreeGrain.cs | 41 + .../DistributedTests.Server/ServerCommand.cs | 1 + .../DistributedTests.Server/ServerRunner.cs | 13 + .../Ping/ITreeGrain.cs | 7 + .../General/RingTests_Standalone.cs | 2 + .../CandidateVertexMaxHeapTests.cs | 47 ++ .../CustomToleranceTests.cs | 223 +++++ .../DefaultToleranceTests.cs | 619 ++++++++++++++ .../FrequencyFilterTests.cs | 140 ++++ .../FrequentEdgeCounterTests.cs | 99 +++ .../ActiveRebalancingTests/OptionsTests.cs | 48 ++ .../RebalancingTestBase.cs | 69 ++ .../TestMessageFilter.cs | 20 + .../GrainDirectoryPartitionTests.cs | 3 + .../ConsistentRingProviderTests.cs | 2 + test/TesterInternal/TesterInternal.csproj | 1 + 73 files changed, 5091 insertions(+), 457 deletions(-) create mode 100644 src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs create mode 100644 src/Orleans.Core/Placement/Rebalancing/IImbalanceToleranceRule.cs create mode 100644 src/Orleans.Core/Placement/Rebalancing/IMessageStatisticsSink.cs create mode 100644 src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs create mode 100644 src/Orleans.Runtime/Hosting/ActiveRebalancingExtensions.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/DefaultImbalanceRule.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs create mode 100644 src/Orleans.Runtime/Utilities/StripedMpscBuffer.cs create mode 100644 test/Benchmarks/Ping/FanoutBenchmark.cs create mode 100644 test/Benchmarks/Ping/TreeGrain.cs create mode 100644 test/Benchmarks/TopK/TopKBenchmark.cs create mode 100644 test/DistributedTests/DistributedTests.Common/GrainInterfaces/ITreeGrain.cs create mode 100644 test/DistributedTests/DistributedTests.Grains/TreeGrain.cs create mode 100644 test/Grains/BenchmarkGrainInterfaces/Ping/ITreeGrain.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/FrequentEdgeCounterTests.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/RebalancingTestBase.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs diff --git a/distributed-tests.yml b/distributed-tests.yml index f4db953da7..16cff5e388 100644 --- a/distributed-tests.yml +++ b/distributed-tests.yml @@ -2,7 +2,7 @@ variables: clusterId: '{{ "now" | date: "%s" }}' serviceId: '{{ "now" | date: "%s" }}' secretSource: KeyVault - framework: net7.0 + framework: net8.0 jobs: server: @@ -10,7 +10,7 @@ jobs: localFolder: Artifacts/DistributedTests/DistributedTests.Server/{{framework}} executable: DistributedTests.Server.exe readyStateText: Orleans Silo started. - framework: net7.0 + framework: net8.0 arguments: "{{configurator}} --clusterId {{clusterId}} --serviceId {{serviceId}} --secretSource {{secretSource}} {{configuratorOptions}}" onConfigure: - if (job.endpoints.Count > 0) { @@ -21,7 +21,7 @@ jobs: localFolder: Artifacts/DistributedTests/DistributedTests.Client/{{framework}} executable: DistributedTests.Client.exe waitForExit: true - framework: net7.0 + framework: net8.0 arguments: "{{command}} --clusterId {{clusterId}} --serviceId {{serviceId}} --secretSource {{secretSource}} {{commandOptions}}" onConfigure: - if (job.endpoints.Count > 0) { @@ -46,6 +46,22 @@ scenarios: requestsPerBlock: 500 duration: 120 commandOptions: "--numWorkers {{numWorkers}} --blocksPerWorker {{blocksPerWorker}} --requestsPerBlock {{requestsPerBlock}} --duration {{duration}}" + fanout: + server: + job: server + variables: + instances: 10 + configurator: SimpleSilo + client: + job: client + variables: + command: fan-out + instances: 1 + numWorkers: 1 + blocksPerWorker: 0 + requestsPerBlock: 50 + duration: 240 + commandOptions: "--numWorkers {{numWorkers}} --blocksPerWorker {{blocksPerWorker}} --requestsPerBlock {{requestsPerBlock}} --duration {{duration}}" streaming: server: job: server @@ -125,6 +141,35 @@ scenarios: duration: 180 commandOptions: "--numWorkers {{numWorkers}} --blocksPerWorker {{blocksPerWorker}} --requestsPerBlock {{requestsPerBlock}} --duration {{duration}}" +counters: +- provider: Microsoft.Orleans + values: + - name: app-requests + measurement: orleans-counter/requests-per-second + description: Request rate + + - name: activation-count + measurement: orleans-counter/grain-activation-count + description: Total number of grains + +results: +# Microsoft.Orleans counters +- name: orleans-counter/requests-per-second + measurement: orleans-counter/requests-per-second + description: Request rate + format: "n0" + aggregate: max + reduce: max +- name: orleans-counter/requests-per-second/95 + measurement: orleans-counter/requests-per-second + description: Request rate + format: "n0" + aggregate: percentile95 + reduce: max +- name: activation-count + measurement: orleans-counter/grain-activation-count + description: Active grains + profiles: local: variables: diff --git a/src/Orleans.Core.Abstractions/IDs/GrainId.cs b/src/Orleans.Core.Abstractions/IDs/GrainId.cs index b7bb724693..1958f457f5 100644 --- a/src/Orleans.Core.Abstractions/IDs/GrainId.cs +++ b/src/Orleans.Core.Abstractions/IDs/GrainId.cs @@ -138,7 +138,7 @@ public static bool TryParse(string? value, IFormatProvider? provider, out GrainI public override bool Equals(object? obj) => obj is GrainId id && Equals(id); /// - public bool Equals(GrainId other) => _type.Equals(other._type) && _key.Equals(other._key); + public bool Equals(GrainId other) => _key.Equals(other._key) && _type.Equals(other._type); /// public override int GetHashCode() => HashCode.Combine(_type, _key); diff --git a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs index 81d575df4b..41181c248d 100644 --- a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs +++ b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs @@ -162,6 +162,11 @@ public static class WellKnownGrainTypeProperties /// Specifies the name of a method used to determine if a request can interleave other requests. /// public const string MayInterleavePredicate = "may-interleave-predicate"; + + /// + /// Whether a grain can be migrated by active-rebalancing or not. + /// + public const string Immovable = "immovable"; } /// diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementAttribute.cs b/src/Orleans.Core.Abstractions/Placement/PlacementAttribute.cs index ae10b343d0..ccb668b3d6 100644 --- a/src/Orleans.Core.Abstractions/Placement/PlacementAttribute.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementAttribute.cs @@ -111,4 +111,16 @@ public sealed class ResourceOptimizedPlacementAttribute : PlacementAttribute base(ResourceOptimizedPlacement.Singleton) { } } + + /// + /// Ensures that when active-rebalancing is enabled, activations of this grain type will not be migrated automatically. + /// + /// Activations can still be migrated by user initiated code. + [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] + public sealed class ImmovableAttribute : Attribute, IGrainPropertiesProviderAttribute + { + /// + public void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) + => properties[WellKnownGrainTypeProperties.Immovable] = "true"; + } } diff --git a/src/Orleans.Core/Configuration/Options/MessagingOptions.cs b/src/Orleans.Core/Configuration/Options/MessagingOptions.cs index fdc47a9eac..90663fb1a8 100644 --- a/src/Orleans.Core/Configuration/Options/MessagingOptions.cs +++ b/src/Orleans.Core/Configuration/Options/MessagingOptions.cs @@ -11,7 +11,7 @@ public abstract class MessagingOptions /// /// The value. /// - private TimeSpan _responseTimeout = TimeSpan.FromSeconds(30); + private TimeSpan _responseTimeout = TimeSpan.FromSeconds(300); /// /// Gets or sets the default timeout before a request is assumed to have failed. diff --git a/src/Orleans.Core/Core/DefaultClientServices.cs b/src/Orleans.Core/Core/DefaultClientServices.cs index a418ffbacf..290c210faa 100644 --- a/src/Orleans.Core/Core/DefaultClientServices.cs +++ b/src/Orleans.Core/Core/DefaultClientServices.cs @@ -25,6 +25,7 @@ using Orleans.Hosting; using System.Reflection; using Microsoft.Extensions.Configuration; +using Orleans.Placement.Rebalancing; namespace Orleans { @@ -114,6 +115,7 @@ public static void AddDefaultServices(IClientBuilder builder) services.AddSingleton(); // Networking + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); diff --git a/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs index dcb3c93e56..16f67c8c97 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/ApplicationRequestInstruments.cs @@ -1,24 +1,43 @@ +#nullable enable using System; using System.Collections.Generic; using System.Diagnostics.Metrics; +using System.Threading; namespace Orleans.Runtime; internal static class ApplicationRequestInstruments { internal static Counter TimedOutRequestsCounter = Instruments.Meter.CreateCounter(InstrumentNames.APP_REQUESTS_TIMED_OUT); + private static long _totalRequests; + private static readonly ObservableCounter RequestsPerSecondCounter = Instruments.Meter.CreateObservableCounter(InstrumentNames.REQUESTS_COMPLETED, () => Volatile.Read(ref _totalRequests)); + + /* private static readonly long[] AppRequestsLatencyHistogramBuckets = new long[] { 1, 2, 4, 6, 8, 10, 50, 100, 200, 400, 800, 1_000, 1_500, 2_000, 5_000, 10_000, 15_000 }; private static readonly HistogramAggregator AppRequestsLatencyHistogramAggregator = new(AppRequestsLatencyHistogramBuckets, Array.Empty>(), value => new ("duration", $"{value}ms")); private static readonly ObservableCounter AppRequestsLatencyHistogramBucket = Instruments.Meter.CreateObservableCounter(InstrumentNames.APP_REQUESTS_LATENCY_HISTOGRAM + "-bucket", AppRequestsLatencyHistogramAggregator.CollectBuckets); private static readonly ObservableCounter AppRequestsLatencyHistogramCount = Instruments.Meter.CreateObservableCounter(InstrumentNames.APP_REQUESTS_LATENCY_HISTOGRAM + "-count", AppRequestsLatencyHistogramAggregator.CollectCount); private static readonly ObservableCounter AppRequestsLatencyHistogramSum = Instruments.Meter.CreateObservableCounter(InstrumentNames.APP_REQUESTS_LATENCY_HISTOGRAM + "-sum", AppRequestsLatencyHistogramAggregator.CollectSum); - + */ + private static readonly Histogram ResponseTimeHistogram = Instruments.Meter.CreateHistogram("response-time", "ms"); internal static void OnAppRequestsEnd(long durationMilliseconds) { + if (RequestsPerSecondCounter.Enabled) + { + Interlocked.Increment(ref _totalRequests); + } + if (ResponseTimeHistogram.Enabled) + { + ResponseTimeHistogram.Record(durationMilliseconds); + } + /* if (AppRequestsLatencyHistogramSum.Enabled) + { AppRequestsLatencyHistogramAggregator.Record(durationMilliseconds); + } + */ } internal static void OnAppRequestsTimedOut() diff --git a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs index 30a5d292df..9cd1d54e4a 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs @@ -12,10 +12,10 @@ internal static class CatalogInstruments internal static Counter ActivationShutdown = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_SHUTDOWN); - internal static void ActiviationShutdownViaCollection() => ActivationShutdown.Add(1, new KeyValuePair("via", "collection")); - internal static void ActiviationShutdownViaDeactivateOnIdle() => ActivationShutdown.Add(1, new KeyValuePair("via", "deactivateOnIdle")); - internal static void ActiviationShutdownViaMigration() => ActivationShutdown.Add(1, new KeyValuePair("via", "migration")); - internal static void ActiviationShutdownViaDeactivateStuckActivation() => ActivationShutdown.Add(1, new KeyValuePair("via", "deactivateStuckActivation")); + internal static void ActivationShutdownViaCollection() => ActivationShutdown.Add(1, new KeyValuePair("via", "collection")); + internal static void ActivationShutdownViaApplication() => ActivationShutdown.Add(1, new KeyValuePair("via", "application")); + internal static void ActivationShutdownViaMigration() => ActivationShutdown.Add(1, new KeyValuePair("via", "migration")); + internal static void ActivationShutdownViaDeactivateStuckActivation() => ActivationShutdown.Add(1, new KeyValuePair("via", "stuck")); internal static Counter NonExistentActivations = Instruments.Meter.CreateCounter(InstrumentNames.CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS); diff --git a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs index f40fe71ff8..8381dfe9ba 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs @@ -3,163 +3,164 @@ namespace Orleans.Runtime; internal static class InstrumentNames { // Networking - public const string NETWORKING_SOCKETS_CLOSED = "orleans-networking-sockets-closed"; - public const string NETWORKING_SOCKETS_OPENED = "orleans-networking-sockets-opened"; + public const string NETWORKING_SOCKETS_CLOSED = "networking-sockets-closed"; + public const string NETWORKING_SOCKETS_OPENED = "networking-sockets-opened"; // Messaging - public const string MESSAGING_SENT_MESSAGES_SIZE = "orleans-messaging-sent-messages-size"; - public const string MESSAGING_RECEIVED_MESSAGES_SIZE = "orleans-messaging-received-messages-size"; - public const string MESSAGING_SENT_BYTES_HEADER = "orleans-messaging-sent-header-size"; - public const string MESSAGING_SENT_FAILED = "orleans-messaging-sent-failed"; - public const string MESSAGING_SENT_DROPPED = "orleans-messaging-sent-dropped"; - public const string MESSAGING_RECEIVED_BYTES_HEADER = "orleans-messaging-received-header-size"; - - public const string MESSAGING_DISPATCHER_RECEIVED = "orleans-messaging-processing-dispatcher-received"; - public const string MESSAGING_DISPATCHER_PROCESSED = "orleans-messaging-processing-dispatcher-processed"; - public const string MESSAGING_DISPATCHER_FORWARDED = "orleans-messaging-processing-dispatcher-forwarded"; - public const string MESSAGING_IMA_RECEIVED = "orleans-messaging-processing-ima-received"; - public const string MESSAGING_IMA_ENQUEUED = "orleans-messaging-processing-ima-enqueued"; - public const string MESSAGING_PROCESSING_ACTIVATION_DATA_ALL = "orleans-messaging-processing-activation-data"; - public const string MESSAGING_PINGS_SENT = "orleans-messaging-pings-sent"; - public const string MESSAGING_PINGS_RECEIVED = "orleans-messaging-pings-received"; - public const string MESSAGING_PINGS_REPLYRECEIVED = "orleans-messaging-pings-reply-received"; - public const string MESSAGING_PINGS_REPLYMISSED = "orleans-messaging-pings-reply-missed"; - public const string MESSAGING_EXPIRED = "orleans-messaging-expired"; - public const string MESSAGING_REJECTED = "orleans-messaging-rejected"; - public const string MESSAGING_REROUTED = "orleans-messaging-rerouted"; - public const string MESSAGING_SENT_LOCALMESSAGES = "orleans-messaging-sent-local"; + public const string MESSAGING_SENT_MESSAGES_SIZE = "messaging-sent-messages-size"; + public const string MESSAGING_RECEIVED_MESSAGES_SIZE = "messaging-received-messages-size"; + public const string MESSAGING_SENT_BYTES_HEADER = "messaging-sent-header-size"; + public const string MESSAGING_SENT_FAILED = "messaging-sent-failed"; + public const string MESSAGING_SENT_DROPPED = "messaging-sent-dropped"; + public const string MESSAGING_RECEIVED_BYTES_HEADER = "messaging-received-header-size"; + + public const string MESSAGING_DISPATCHER_RECEIVED = "messaging-processing-dispatcher-received"; + public const string MESSAGING_DISPATCHER_PROCESSED = "messaging-processing-dispatcher-processed"; + public const string MESSAGING_DISPATCHER_FORWARDED = "messaging-processing-dispatcher-forwarded"; + public const string MESSAGING_IMA_RECEIVED = "messaging-processing-ima-received"; + public const string MESSAGING_IMA_ENQUEUED = "messaging-processing-ima-enqueued"; + public const string MESSAGING_PROCESSING_ACTIVATION_DATA_ALL = "messaging-processing-activation-data"; + public const string MESSAGING_PINGS_SENT = "messaging-pings-sent"; + public const string MESSAGING_PINGS_RECEIVED = "messaging-pings-received"; + public const string MESSAGING_PINGS_REPLYRECEIVED = "messaging-pings-reply-received"; + public const string MESSAGING_PINGS_REPLYMISSED = "messaging-pings-reply-missed"; + public const string MESSAGING_EXPIRED = "messaging-expired"; + public const string MESSAGING_REJECTED = "messaging-rejected"; + public const string MESSAGING_REROUTED = "messaging-rerouted"; + public const string MESSAGING_SENT_LOCALMESSAGES = "messaging-sent-local"; // Gateway - public const string GATEWAY_CONNECTED_CLIENTS = "orleans-gateway-connected-clients"; - public const string GATEWAY_SENT = "orleans-gateway-sent"; - public const string GATEWAY_RECEIVED = "orleans-gateway-received"; - public const string GATEWAY_LOAD_SHEDDING = "orleans-gateway-load-shedding"; + public const string GATEWAY_CONNECTED_CLIENTS = "gateway-connected-clients"; + public const string GATEWAY_SENT = "gateway-sent"; + public const string GATEWAY_RECEIVED = "gateway-received"; + public const string GATEWAY_LOAD_SHEDDING = "gateway-load-shedding"; // Runtime - public const string SCHEDULER_NUM_LONG_RUNNING_TURNS = "orleans-scheduler-long-running-turns"; + public const string SCHEDULER_NUM_LONG_RUNNING_TURNS = "scheduler-long-running-turns"; // Catalog - public const string CATALOG_ACTIVATION_COUNT = "orleans-catalog-activations"; - public const string CATALOG_ACTIVATION_WORKING_SET = "orleans-catalog-activation-working-set"; - public const string CATALOG_ACTIVATION_CREATED = "orleans-catalog-activation-created"; - public const string CATALOG_ACTIVATION_DESTROYED = "orleans-catalog-activation-destroyed"; - public const string CATALOG_ACTIVATION_FAILED_TO_ACTIVATE = "orleans-catalog-activation-failed-to-activate"; - public const string CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS = "orleans-catalog-activation-collections"; - public const string CATALOG_ACTIVATION_SHUTDOWN = "orleans-catalog-activation-shutdown"; - public const string CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS = "orleans-catalog-activation-non-existent"; - public const string CATALOG_ACTIVATION_CONCURRENT_REGISTRATION_ATTEMPTS = "orleans-catalog-activation-concurrent-registration-attempts"; + public const string CATALOG_ACTIVATION_COUNT = "activation-count"; + public const string CATALOG_ACTIVATION_WORKING_SET = "catalog-activation-working-set"; + public const string CATALOG_ACTIVATION_CREATED = "catalog-activation-created"; + public const string CATALOG_ACTIVATION_DESTROYED = "catalog-activation-destroyed"; + public const string CATALOG_ACTIVATION_FAILED_TO_ACTIVATE = "catalog-activation-failed-to-activate"; + public const string CATALOG_ACTIVATION_COLLECTION_NUMBER_OF_COLLECTIONS = "catalog-activation-collections"; + public const string CATALOG_ACTIVATION_SHUTDOWN = "catalog-activation-shutdown"; + public const string CATALOG_ACTIVATION_NON_EXISTENT_ACTIVATIONS = "catalog-activation-non-existent"; + public const string CATALOG_ACTIVATION_CONCURRENT_REGISTRATION_ATTEMPTS = "catalog-activation-concurrent-registration-attempts"; // Directory // not used... - public const string DIRECTORY_LOOKUPS_LOCAL_ISSUED = "orleans-directory-lookups-local-issued"; + public const string DIRECTORY_LOOKUPS_LOCAL_ISSUED = "directory-lookups-local-issued"; // not used... - public const string DIRECTORY_LOOKUPS_LOCAL_SUCCESSES = "orleans-directory-lookups-local-successes"; - public const string DIRECTORY_LOOKUPS_FULL_ISSUED = "orleans-directory-lookups-full-issued"; - public const string DIRECTORY_LOOKUPS_REMOTE_SENT = "orleans-directory-lookups-remote-sent"; - public const string DIRECTORY_LOOKUPS_REMOTE_RECEIVED = "orleans-directory-lookups-remote-received"; - public const string DIRECTORY_LOOKUPS_LOCALDIRECTORY_ISSUED = "orleans-directory-lookups-local-directory-issued"; - public const string DIRECTORY_LOOKUPS_LOCALDIRECTORY_SUCCESSES = "orleans-directory-lookups-local-directory-successes"; + public const string DIRECTORY_LOOKUPS_LOCAL_SUCCESSES = "directory-lookups-local-successes"; + public const string DIRECTORY_LOOKUPS_FULL_ISSUED = "directory-lookups-full-issued"; + public const string DIRECTORY_LOOKUPS_REMOTE_SENT = "directory-lookups-remote-sent"; + public const string DIRECTORY_LOOKUPS_REMOTE_RECEIVED = "directory-lookups-remote-received"; + public const string DIRECTORY_LOOKUPS_LOCALDIRECTORY_ISSUED = "directory-lookups-local-directory-issued"; + public const string DIRECTORY_LOOKUPS_LOCALDIRECTORY_SUCCESSES = "directory-lookups-local-directory-successes"; // not used - public const string DIRECTORY_LOOKUPS_CACHE_ISSUED = "orleans-directory-lookups-cache-issued"; + public const string DIRECTORY_LOOKUPS_CACHE_ISSUED = "directory-lookups-cache-issued"; // not used - public const string DIRECTORY_LOOKUPS_CACHE_SUCCESSES = "orleans-directory-lookups-cache-successes"; - public const string DIRECTORY_VALIDATIONS_CACHE_SENT = "orleans-directory-validations-cache-sent"; - public const string DIRECTORY_VALIDATIONS_CACHE_RECEIVED = "orleans-directory-validations-cache-received"; - public const string DIRECTORY_PARTITION_SIZE = "orleans-directory-partition-size"; - public const string DIRECTORY_CACHE_SIZE = "orleans-directory-cache-size"; - public const string DIRECTORY_RING_RINGSIZE = "orleans-directory-ring-size"; - public const string DIRECTORY_RING_MYPORTION_RINGDISTANCE = "orleans-directory-ring-local-portion-distance"; - public const string DIRECTORY_RING_MYPORTION_RINGPERCENTAGE = "orleans-directory-ring-local-portion-percentage"; - public const string DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE = "orleans-directory-ring-local-portion-average-percentage"; - public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_ISSUED = "orleans-directory-registrations-single-act-issued"; - public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL = "orleans-directory-registrations-single-act-local"; - public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT = "orleans-directory-registrations-single-act-remote-sent"; - public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED = "orleans-directory-registrations-single-act-remote-received"; - public const string DIRECTORY_UNREGISTRATIONS_ISSUED = "orleans-directory-unregistrations-issued"; - public const string DIRECTORY_UNREGISTRATIONS_LOCAL = "orleans-directory-unregistrations-local"; - public const string DIRECTORY_UNREGISTRATIONS_REMOTE_SENT = "orleans-directory-unregistrations-remote-sent"; - public const string DIRECTORY_UNREGISTRATIONS_REMOTE_RECEIVED = "orleans-directory-unregistrations-remote-received"; - public const string DIRECTORY_UNREGISTRATIONS_MANY_ISSUED = "orleans-directory-unregistrations-many-issued"; - public const string DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_SENT = "orleans-directory-unregistrations-many-remote-sent"; - public const string DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_RECEIVED = "orleans-directory-unregistrations-many-remote-received"; + public const string DIRECTORY_LOOKUPS_CACHE_SUCCESSES = "directory-lookups-cache-successes"; + public const string DIRECTORY_VALIDATIONS_CACHE_SENT = "directory-validations-cache-sent"; + public const string DIRECTORY_VALIDATIONS_CACHE_RECEIVED = "directory-validations-cache-received"; + public const string DIRECTORY_PARTITION_SIZE = "directory-partition-size"; + public const string DIRECTORY_CACHE_SIZE = "directory-cache-size"; + public const string DIRECTORY_RING_RINGSIZE = "directory-ring-size"; + public const string DIRECTORY_RING_MYPORTION_RINGDISTANCE = "directory-ring-local-portion-distance"; + public const string DIRECTORY_RING_MYPORTION_RINGPERCENTAGE = "directory-ring-local-portion-percentage"; + public const string DIRECTORY_RING_MYPORTION_AVERAGERINGPERCENTAGE = "directory-ring-local-portion-average-percentage"; + public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_ISSUED = "directory-registrations-single-act-issued"; + public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_LOCAL = "directory-registrations-single-act-local"; + public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_SENT = "directory-registrations-single-act-remote-sent"; + public const string DIRECTORY_REGISTRATIONS_SINGLE_ACT_REMOTE_RECEIVED = "directory-registrations-single-act-remote-received"; + public const string DIRECTORY_UNREGISTRATIONS_ISSUED = "directory-unregistrations-issued"; + public const string DIRECTORY_UNREGISTRATIONS_LOCAL = "directory-unregistrations-local"; + public const string DIRECTORY_UNREGISTRATIONS_REMOTE_SENT = "directory-unregistrations-remote-sent"; + public const string DIRECTORY_UNREGISTRATIONS_REMOTE_RECEIVED = "directory-unregistrations-remote-received"; + public const string DIRECTORY_UNREGISTRATIONS_MANY_ISSUED = "directory-unregistrations-many-issued"; + public const string DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_SENT = "directory-unregistrations-many-remote-sent"; + public const string DIRECTORY_UNREGISTRATIONS_MANY_REMOTE_RECEIVED = "directory-unregistrations-many-remote-received"; // ConsistentRing - public const string CONSISTENTRING_SIZE = "orleans-consistent-ring-size"; - public const string CONSISTENTRING_LOCAL_SIZE_PERCENTAGE = "orleans-consistent-ring-range-percentage-local"; - public const string CONSISTENTRING_AVERAGE_SIZE_PERCENTAGE = "orleans-consistent-ring-range-percentage-average"; + public const string CONSISTENTRING_SIZE = "consistent-ring-size"; + public const string CONSISTENTRING_LOCAL_SIZE_PERCENTAGE = "consistent-ring-range-percentage-local"; + public const string CONSISTENTRING_AVERAGE_SIZE_PERCENTAGE = "consistent-ring-range-percentage-average"; // Watchdog - public const string WATCHDOG_NUM_HEALTH_CHECKS = "orleans-watchdog-health-checks"; - public const string WATCHDOG_NUM_FAILED_HEALTH_CHECKS = "orleans-watchdog-health-checks-failed"; + public const string WATCHDOG_NUM_HEALTH_CHECKS = "watchdog-health-checks"; + public const string WATCHDOG_NUM_FAILED_HEALTH_CHECKS = "watchdog-health-checks-failed"; // Client - public const string CLIENT_CONNECTED_GATEWAY_COUNT = "orleans-client-connected-gateways"; + public const string CLIENT_CONNECTED_GATEWAY_COUNT = "client-connected-gateways"; // Misc - public const string GRAIN_COUNTS = "orleans-grains"; - public const string SYSTEM_TARGET_COUNTS = "orleans-system-targets"; + public const string GRAIN_COUNTS = "grains"; + public const string SYSTEM_TARGET_COUNTS = "system-targets"; // App requests - public const string APP_REQUESTS_LATENCY_HISTOGRAM = "orleans-app-requests-latency"; - public const string APP_REQUESTS_TIMED_OUT = "orleans-app-requests-timedout"; + public const string REQUESTS_COMPLETED = "app-requests"; + public const string APP_REQUESTS_LATENCY_HISTOGRAM = "app-requests-latency"; + public const string APP_REQUESTS_TIMED_OUT = "app-requests-timedout"; // Reminders - public const string REMINDERS_TARDINESS = "orleans-reminders-tardiness"; - public const string REMINDERS_NUMBER_ACTIVE_REMINDERS = "orleans-reminders-active"; - public const string REMINDERS_COUNTERS_TICKS_DELIVERED = "orleans-reminders-ticks-delivered"; + public const string REMINDERS_TARDINESS = "reminders-tardiness"; + public const string REMINDERS_NUMBER_ACTIVE_REMINDERS = "reminders-active"; + public const string REMINDERS_COUNTERS_TICKS_DELIVERED = "reminders-ticks-delivered"; // Storage - public const string STORAGE_READ_ERRORS = "orleans-storage-read-errors"; - public const string STORAGE_WRITE_ERRORS = "orleans-storage-write-errors"; - public const string STORAGE_CLEAR_ERRORS = "orleans-storage-clear-errors"; - public const string STORAGE_READ_LATENCY = "orleans-storage-read-latency"; - public const string STORAGE_WRITE_LATENCY = "orleans-storage-write-latency"; - public const string STORAGE_CLEAR_LATENCY = "orleans-storage-clear-latency"; + public const string STORAGE_READ_ERRORS = "storage-read-errors"; + public const string STORAGE_WRITE_ERRORS = "storage-write-errors"; + public const string STORAGE_CLEAR_ERRORS = "storage-clear-errors"; + public const string STORAGE_READ_LATENCY = "storage-read-latency"; + public const string STORAGE_WRITE_LATENCY = "storage-write-latency"; + public const string STORAGE_CLEAR_LATENCY = "storage-clear-latency"; // Streams - public const string STREAMS_PUBSUB_PRODUCERS_ADDED = "orleans-streams-pubsub-producers-added"; - public const string STREAMS_PUBSUB_PRODUCERS_REMOVED = "orleans-streams-pubsub-producers-removed"; - public const string STREAMS_PUBSUB_PRODUCERS_TOTAL = "orleans-streams-pubsub-producers"; - public const string STREAMS_PUBSUB_CONSUMERS_ADDED = "orleans-streams-pubsub-consumers-added"; - public const string STREAMS_PUBSUB_CONSUMERS_REMOVED = "orleans-streams-pubsub-consumers-removed"; - public const string STREAMS_PUBSUB_CONSUMERS_TOTAL = "orleans-streams-pubsub-consumers"; - - public const string STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS = "orleans-streams-persistent-stream-pulling-agents"; - public const string STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES = "orleans-streams-persistent-stream-messages-read"; - public const string STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES = "orleans-streams-persistent-stream-messages-sent"; - public const string STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE = "orleans-streams-persistent-stream-pubsub-cache-size"; - - public const string STREAMS_QUEUE_INITIALIZATION_FAILURES = "orleans-streams-queue-initialization-failures"; - public const string STREAMS_QUEUE_INITIALIZATION_DURATION = "orleans-streams-queue-initialization-duration"; - public const string STREAMS_QUEUE_INITIALIZATION_EXCEPTIONS = "orleans-streams-queue-initialization-exceptions"; - public const string STREAMS_QUEUE_READ_FAILURES = "orleans-streams-queue-read-failures"; - public const string STREAMS_QUEUE_READ_DURATION = "orleans-streams-queue-read-duration"; - public const string STREAMS_QUEUE_READ_EXCEPTIONS = "orleans-streams-queue-read-exceptions"; - public const string STREAMS_QUEUE_SHUTDOWN_FAILURES = "orleans-streams-queue-shutdown-failures"; - public const string STREAMS_QUEUE_SHUTDOWN_DURATION = "orleans-streams-queue-shutdown-duration"; - public const string STREAMS_QUEUE_SHUTDOWN_EXCEPTIONS = "orleans-streams-queue-shutdown-exceptions"; - public const string STREAMS_QUEUE_MESSAGES_RECEIVED = "orleans-streams-queue-messages-received"; - public const string STREAMS_QUEUE_OLDEST_MESSAGE_ENQUEUE_AGE = "orleans-streams-queue-oldest-message-enqueue-age"; - public const string STREAMS_QUEUE_NEWEST_MESSAGE_ENQUEUE_AGE = "orleans-streams-queue-newest-message-enqueue-age"; - - public const string STREAMS_BLOCK_POOL_TOTAL_MEMORY = "orleans-streams-block-pool-total-memory"; - public const string STREAMS_BLOCK_POOL_AVAILABLE_MEMORY = "orleans-streams-block-pool-available-memory"; - public const string STREAMS_BLOCK_POOL_CLAIMED_MEMORY = "orleans-streams-block-pool-claimed-memory"; - public const string STREAMS_BLOCK_POOL_RELEASED_MEMORY = "orleans-streams-block-pool-released-memory"; - public const string STREAMS_BLOCK_POOL_ALLOCATED_MEMORY = "orleans-streams-block-pool-allocated-memory"; - - public const string STREAMS_QUEUE_CACHE_SIZE = "orleans-streams-queue-cache-size"; - public const string STREAMS_QUEUE_CACHE_LENGTH = "orleans-streams-queue-cache-length"; - public const string STREAMS_QUEUE_CACHE_MESSAGES_ADDED = "orleans-streams-queue-cache-messages-added"; - public const string STREAMS_QUEUE_CACHE_MESSAGES_PURGED = "orleans-streams-queue-cache-messages-purged"; - public const string STREAMS_QUEUE_CACHE_MEMORY_ALLOCATED = "orleans-streams-queue-cache-memory-allocated"; - public const string STREAMS_QUEUE_CACHE_MEMORY_RELEASED = "orleans-streams-queue-cache-memory-released"; - public const string STREAMS_QUEUE_CACHE_OLDEST_TO_NEWEST_DURATION = "orleans-streams-queue-cache-oldest-to-newest-duration"; - public const string STREAMS_QUEUE_CACHE_OLDEST_AGE = "orleans-streams-queue-cache-oldest-age"; - public const string STREAMS_QUEUE_CACHE_PRESSURE = "orleans-streams-queue-cache-pressure"; - public const string STREAMS_QUEUE_CACHE_UNDER_PRESSURE = "orleans-streams-queue-cache-under-pressure"; - public const string STREAMS_QUEUE_CACHE_PRESSURE_CONTRIBUTION_COUNT = "orleans-streams-queue-cache-pressure-contribution-count"; - - public const string RUNTIME_MEMORY_TOTAL_PHYSICAL_MEMORY_MB = "orleans-runtime-total-physical-memory"; - public const string RUNTIME_MEMORY_AVAILABLE_MEMORY_MB = "orleans-runtime-available-memory"; + public const string STREAMS_PUBSUB_PRODUCERS_ADDED = "streams-pubsub-producers-added"; + public const string STREAMS_PUBSUB_PRODUCERS_REMOVED = "streams-pubsub-producers-removed"; + public const string STREAMS_PUBSUB_PRODUCERS_TOTAL = "streams-pubsub-producers"; + public const string STREAMS_PUBSUB_CONSUMERS_ADDED = "streams-pubsub-consumers-added"; + public const string STREAMS_PUBSUB_CONSUMERS_REMOVED = "streams-pubsub-consumers-removed"; + public const string STREAMS_PUBSUB_CONSUMERS_TOTAL = "streams-pubsub-consumers"; + + public const string STREAMS_PERSISTENT_STREAM_NUM_PULLING_AGENTS = "streams-persistent-stream-pulling-agents"; + public const string STREAMS_PERSISTENT_STREAM_NUM_READ_MESSAGES = "streams-persistent-stream-messages-read"; + public const string STREAMS_PERSISTENT_STREAM_NUM_SENT_MESSAGES = "streams-persistent-stream-messages-sent"; + public const string STREAMS_PERSISTENT_STREAM_PUBSUB_CACHE_SIZE = "streams-persistent-stream-pubsub-cache-size"; + + public const string STREAMS_QUEUE_INITIALIZATION_FAILURES = "streams-queue-initialization-failures"; + public const string STREAMS_QUEUE_INITIALIZATION_DURATION = "streams-queue-initialization-duration"; + public const string STREAMS_QUEUE_INITIALIZATION_EXCEPTIONS = "streams-queue-initialization-exceptions"; + public const string STREAMS_QUEUE_READ_FAILURES = "streams-queue-read-failures"; + public const string STREAMS_QUEUE_READ_DURATION = "streams-queue-read-duration"; + public const string STREAMS_QUEUE_READ_EXCEPTIONS = "streams-queue-read-exceptions"; + public const string STREAMS_QUEUE_SHUTDOWN_FAILURES = "streams-queue-shutdown-failures"; + public const string STREAMS_QUEUE_SHUTDOWN_DURATION = "streams-queue-shutdown-duration"; + public const string STREAMS_QUEUE_SHUTDOWN_EXCEPTIONS = "streams-queue-shutdown-exceptions"; + public const string STREAMS_QUEUE_MESSAGES_RECEIVED = "streams-queue-messages-received"; + public const string STREAMS_QUEUE_OLDEST_MESSAGE_ENQUEUE_AGE = "streams-queue-oldest-message-enqueue-age"; + public const string STREAMS_QUEUE_NEWEST_MESSAGE_ENQUEUE_AGE = "streams-queue-newest-message-enqueue-age"; + + public const string STREAMS_BLOCK_POOL_TOTAL_MEMORY = "streams-block-pool-total-memory"; + public const string STREAMS_BLOCK_POOL_AVAILABLE_MEMORY = "streams-block-pool-available-memory"; + public const string STREAMS_BLOCK_POOL_CLAIMED_MEMORY = "streams-block-pool-claimed-memory"; + public const string STREAMS_BLOCK_POOL_RELEASED_MEMORY = "streams-block-pool-released-memory"; + public const string STREAMS_BLOCK_POOL_ALLOCATED_MEMORY = "streams-block-pool-allocated-memory"; + + public const string STREAMS_QUEUE_CACHE_SIZE = "streams-queue-cache-size"; + public const string STREAMS_QUEUE_CACHE_LENGTH = "streams-queue-cache-length"; + public const string STREAMS_QUEUE_CACHE_MESSAGES_ADDED = "streams-queue-cache-messages-added"; + public const string STREAMS_QUEUE_CACHE_MESSAGES_PURGED = "streams-queue-cache-messages-purged"; + public const string STREAMS_QUEUE_CACHE_MEMORY_ALLOCATED = "streams-queue-cache-memory-allocated"; + public const string STREAMS_QUEUE_CACHE_MEMORY_RELEASED = "streams-queue-cache-memory-released"; + public const string STREAMS_QUEUE_CACHE_OLDEST_TO_NEWEST_DURATION = "streams-queue-cache-oldest-to-newest-duration"; + public const string STREAMS_QUEUE_CACHE_OLDEST_AGE = "streams-queue-cache-oldest-age"; + public const string STREAMS_QUEUE_CACHE_PRESSURE = "streams-queue-cache-pressure"; + public const string STREAMS_QUEUE_CACHE_UNDER_PRESSURE = "streams-queue-cache-under-pressure"; + public const string STREAMS_QUEUE_CACHE_PRESSURE_CONTRIBUTION_COUNT = "streams-queue-cache-pressure-contribution-count"; + + public const string RUNTIME_MEMORY_TOTAL_PHYSICAL_MEMORY_MB = "runtime-total-physical-memory"; + public const string RUNTIME_MEMORY_AVAILABLE_MEMORY_MB = "runtime-available-memory"; } diff --git a/src/Orleans.Core/Diagnostics/Metrics/MessagingInstruments.cs b/src/Orleans.Core/Diagnostics/Metrics/MessagingInstruments.cs index 79c4409075..a8f479ea60 100644 --- a/src/Orleans.Core/Diagnostics/Metrics/MessagingInstruments.cs +++ b/src/Orleans.Core/Diagnostics/Metrics/MessagingInstruments.cs @@ -46,83 +46,69 @@ internal enum Phase internal static void OnMessageExpired(Phase phase) { - ExpiredMessagesCounter.Add(1, new KeyValuePair("Phase", phase)); + ExpiredMessagesCounter.Add(1, new KeyValuePair("phase", phase)); } internal static void OnPingSend(SiloAddress destination) { - PingSendCounter.Add(1, new KeyValuePair("Destination", destination.ToString())); + PingSendCounter.Add(1, new KeyValuePair("destination", destination)); } internal static void OnPingReceive(SiloAddress destination) { - PingReceivedCounter.Add(1, new KeyValuePair("Destination", destination.ToString())); + PingReceivedCounter.Add(1, new KeyValuePair("destination", destination)); } internal static void OnPingReplyReceived(SiloAddress replier) { - PingReplyReceivedCounter.Add(1, new KeyValuePair("Destination", replier.ToString())); + PingReplyReceivedCounter.Add(1, new KeyValuePair("destination", replier)); } internal static void OnPingReplyMissed(SiloAddress replier) { - PingReplyMissedCounter.Add(1, new KeyValuePair("Destination", replier.ToString())); + PingReplyMissedCounter.Add(1, new KeyValuePair("destination", replier)); } internal static void OnFailedSentMessage(Message msg) { if (msg == null || !msg.HasDirection) return; - FailedSentMessagesCounter.Add(1, new KeyValuePair("Direction", msg.Direction.ToString())); + FailedSentMessagesCounter.Add(1, new KeyValuePair("direction", msg.Direction.ToString())); } internal static void OnDroppedSentMessage(Message msg) { if (msg == null || !msg.HasDirection) return; - DroppedSentMessagesCounter.Add(1, new KeyValuePair("Direction", msg.Direction.ToString())); + DroppedSentMessagesCounter.Add(1, new KeyValuePair("direction", msg.Direction.ToString())); } internal static void OnRejectedMessage(Message msg) { if (msg == null || !msg.HasDirection) return; - RejectedMessagesCounter.Add(1, new KeyValuePair("Direction", msg.Direction.ToString())); + RejectedMessagesCounter.Add(1, new KeyValuePair("direction", msg.Direction.ToString())); } internal static void OnMessageReRoute(Message msg) { - ReroutedMessagesCounter.Add(1, new KeyValuePair("Direction", msg.Direction.ToString())); + ReroutedMessagesCounter.Add(1, new KeyValuePair("direction", msg.Direction.ToString())); } - internal static void OnMessageReceive(Message msg, int numTotalBytes, int headerBytes, ConnectionDirection connectionDirection, SiloAddress remoteSiloAddress = null) + internal static void OnMessageReceive(Message msg, int numTotalBytes, int headerBytes) { if (MessageReceivedSizeHistogram.Enabled) { - if (remoteSiloAddress != null) - { - MessageReceivedSizeHistogram.Record(numTotalBytes, new KeyValuePair("ConnectionDirection", connectionDirection.ToString()), new KeyValuePair("MessageDirection", msg.Direction.ToString()), new KeyValuePair("silo", remoteSiloAddress)); - } - else - { - MessageReceivedSizeHistogram.Record(numTotalBytes, new KeyValuePair("ConnectionDirection", connectionDirection.ToString()), new KeyValuePair("MessageDirection", msg.Direction.ToString())); - } + MessageReceivedSizeHistogram.Record(numTotalBytes, new KeyValuePair("source", msg.SendingSilo), new KeyValuePair("destination", msg.TargetSilo)); } Interlocked.Add(ref _headerBytesReceived, headerBytes); } - internal static void OnMessageSend(Message msg, int numTotalBytes, int headerBytes, ConnectionDirection connectionDirection, SiloAddress remoteSiloAddress = null) + internal static void OnMessageSend(Message msg, int numTotalBytes, int headerBytes) { Debug.Assert(numTotalBytes >= 0, $"OnMessageSend(numTotalBytes={numTotalBytes})"); if (MessageSentSizeHistogram.Enabled) { - if (remoteSiloAddress != null) - { - MessageSentSizeHistogram.Record(numTotalBytes, new KeyValuePair("ConnectionDirection", connectionDirection.ToString()), new KeyValuePair("MessageDirection", msg.Direction.ToString()), new KeyValuePair("silo", remoteSiloAddress)); - } - else - { - MessageSentSizeHistogram.Record(numTotalBytes, new KeyValuePair("ConnectionDirection", connectionDirection.ToString()), new KeyValuePair("MessageDirection", msg.Direction.ToString())); - } + MessageSentSizeHistogram.Record(numTotalBytes, new KeyValuePair("source", msg.SendingSilo), new KeyValuePair("destination", msg.TargetSilo)); } Interlocked.Add(ref _headerBytesSent, headerBytes); diff --git a/src/Orleans.Core/Messaging/Message.cs b/src/Orleans.Core/Messaging/Message.cs index ce6d13400d..88b45548b0 100644 --- a/src/Orleans.Core/Messaging/Message.cs +++ b/src/Orleans.Core/Messaging/Message.cs @@ -74,7 +74,8 @@ public Directions Direction public bool HasDirection => _headers.Direction != Directions.None; - public bool IsFullyAddressed => TargetSilo is not null && !TargetGrain.IsDefault; + public bool IsSenderFullyAddressed => SendingSilo is not null && !SendingGrain.IsDefault; + public bool IsTargetFullyAddressed => TargetSilo is not null && !TargetGrain.IsDefault; public bool IsExpired => _timeToExpiry is { IsDefault: false, ElapsedMilliseconds: > 0 }; diff --git a/src/Orleans.Core/Networking/ClientOutboundConnection.cs b/src/Orleans.Core/Networking/ClientOutboundConnection.cs index baaf0856de..91c57ddc4c 100644 --- a/src/Orleans.Core/Networking/ClientOutboundConnection.cs +++ b/src/Orleans.Core/Networking/ClientOutboundConnection.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging; using Orleans.Configuration; using Orleans.Messaging; +using Orleans.Placement.Rebalancing; namespace Orleans.Runtime.Messaging { @@ -45,12 +46,12 @@ internal sealed class ClientOutboundConnection : Connection protected override void RecordMessageReceive(Message msg, int numTotalBytes, int headerBytes) { - MessagingInstruments.OnMessageReceive(msg, numTotalBytes, headerBytes, ConnectionDirection, RemoteSiloAddress); + MessagingInstruments.OnMessageReceive(msg, numTotalBytes, headerBytes); } protected override void RecordMessageSend(Message msg, int numTotalBytes, int headerBytes) { - MessagingInstruments.OnMessageSend(msg, numTotalBytes, headerBytes, ConnectionDirection, RemoteSiloAddress); + MessagingInstruments.OnMessageSend(msg, numTotalBytes, headerBytes); } protected override void OnReceivedMessage(Message message) diff --git a/src/Orleans.Core/Networking/ClientOutboundConnectionFactory.cs b/src/Orleans.Core/Networking/ClientOutboundConnectionFactory.cs index a4349957e3..038e75526a 100644 --- a/src/Orleans.Core/Networking/ClientOutboundConnectionFactory.cs +++ b/src/Orleans.Core/Networking/ClientOutboundConnectionFactory.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Options; using Orleans.Configuration; using Orleans.Messaging; +using Orleans.Placement.Rebalancing; namespace Orleans.Runtime.Messaging { diff --git a/src/Orleans.Core/Networking/Connection.cs b/src/Orleans.Core/Networking/Connection.cs index f4d9f24179..3205ac1e96 100644 --- a/src/Orleans.Core/Networking/Connection.cs +++ b/src/Orleans.Core/Networking/Connection.cs @@ -12,6 +12,7 @@ using Microsoft.Extensions.ObjectPool; using Orleans.Configuration; using Orleans.Messaging; +using Orleans.Placement.Rebalancing; using Orleans.Serialization.Invocation; namespace Orleans.Runtime.Messaging @@ -274,9 +275,7 @@ public virtual void Send(Message message) protected abstract void RecordMessageReceive(Message msg, int numTotalBytes, int headerBytes); protected abstract void RecordMessageSend(Message msg, int numTotalBytes, int headerBytes); - protected abstract void OnReceivedMessage(Message message); - protected abstract void OnSendMessageFailure(Message message, string error); private async Task ProcessIncoming() @@ -355,6 +354,7 @@ private async Task ProcessOutgoing() Exception error = default; var serializer = this.shared.ServiceProvider.GetRequiredService(); + var messageStatisticsSink = this.shared.MessageStatisticsSink; try { var output = this._transport.Output; @@ -376,6 +376,7 @@ private async Task ProcessOutgoing() inflight.Add(message); var (headerLength, bodyLength) = serializer.Write(output, message); RecordMessageSend(message, headerLength + bodyLength, headerLength); + messageStatisticsSink.RecordMessage(message); message = null; } } diff --git a/src/Orleans.Core/Networking/ConnectionShared.cs b/src/Orleans.Core/Networking/ConnectionShared.cs index a6b21f76cc..fde60cf471 100644 --- a/src/Orleans.Core/Networking/ConnectionShared.cs +++ b/src/Orleans.Core/Networking/ConnectionShared.cs @@ -1,24 +1,19 @@ -using System; +using System; +using Orleans.Placement.Rebalancing; namespace Orleans.Runtime.Messaging { - internal sealed class ConnectionCommon + internal sealed class ConnectionCommon( + IServiceProvider serviceProvider, + MessageFactory messageFactory, + MessagingTrace messagingTrace, + NetworkingTrace networkingTrace, + IMessageStatisticsSink messageStatisticsSink) { - public ConnectionCommon( - IServiceProvider serviceProvider, - MessageFactory messageFactory, - MessagingTrace messagingTrace, - NetworkingTrace networkingTrace) - { - this.ServiceProvider = serviceProvider; - this.MessageFactory = messageFactory; - this.MessagingTrace = messagingTrace; - this.NetworkingTrace = networkingTrace; - } - - public MessageFactory MessageFactory { get; } - public IServiceProvider ServiceProvider { get; } - public NetworkingTrace NetworkingTrace { get; } - public MessagingTrace MessagingTrace { get; } + public MessageFactory MessageFactory { get; } = messageFactory; + public IServiceProvider ServiceProvider { get; } = serviceProvider; + public NetworkingTrace NetworkingTrace { get; } = networkingTrace; + public IMessageStatisticsSink MessageStatisticsSink { get; } = messageStatisticsSink; + public MessagingTrace MessagingTrace { get; } = messagingTrace; } } diff --git a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs new file mode 100644 index 0000000000..3727bc1ba2 --- /dev/null +++ b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs @@ -0,0 +1,190 @@ +using System.Threading.Tasks; +using System.Collections.Immutable; +using Orleans.Runtime; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System; + +namespace Orleans.Placement.Rebalancing; + +[Alias("IActivationRebalancerSystemTarget")] +internal interface IActivationRebalancerSystemTarget : ISystemTarget +{ + static IActivationRebalancerSystemTarget GetReference(IGrainFactory grainFactory, SiloAddress targetSilo) + => grainFactory.GetGrain(SystemTargetGrainId.Create(Constants.ActivationRebalancerType, targetSilo).GrainId); + + ValueTask TriggerExchangeRequest(); + + ValueTask AcceptExchangeRequest(AcceptExchangeRequest request); + + /// + /// For use in testing only! + /// + ValueTask ResetCounters(); + + /// + /// For use in testing only! + /// + ValueTask GetActivationCount(); + + /// + /// For use in testing only! + /// + ValueTask SetActivationCountOffset(int activationCountOffset); +} + +// We use a readonly struct so that we can fully decouple the message-passing and potentially modifications to the Silo fields. +/// +/// Data structure representing a 'communication edge' between a source and target. +/// +[GenerateSerializer, Immutable, DebuggerDisplay("Source: [{Source.Id} - {Source.Silo}] | Target: [{Target.Id} - {Target.Silo}]")] +internal readonly struct Edge(EdgeVertex source, EdgeVertex target) : IEquatable +{ + [Id(0)] + public EdgeVertex Source { get; } = source; + + [Id(1)] + public EdgeVertex Target { get; } = target; + + public static bool operator ==(Edge left, Edge right) => left.Equals(right); + public static bool operator !=(Edge left, Edge right) => !left.Equals(right); + + public override bool Equals([NotNullWhen(true)] object obj) => obj is Edge other && Equals(other); + public bool Equals(Edge other) => Source == other.Source && Target == other.Target; + + public override int GetHashCode() => HashCode.Combine(Source, Target); + + /// + /// Returns a copy of this but with flipped sources and targets. + /// + public Edge Flip() => new(source: Target, target: Source); + + /// + /// Checks if any of the is part of this counter. + /// + public readonly bool ContainsAny(ImmutableArray grainIds) + { + foreach (var grainId in grainIds) + { + if (Source.Id == grainId || Target.Id == grainId) + { + return true; + } + } + + return false; + } +} + +/// +/// Data structure representing one side of a . +/// +[GenerateSerializer, Immutable] +public readonly struct EdgeVertex( + GrainId id, + SiloAddress silo, + bool isMigratable) : IEquatable +{ + [Id(0)] + public readonly GrainId Id = id; + + [Id(1)] + public readonly SiloAddress Silo = silo; + + [Id(2)] + public readonly bool IsMigratable = isMigratable; + + public static bool operator ==(EdgeVertex left, EdgeVertex right) => left.Equals(right); + public static bool operator !=(EdgeVertex left, EdgeVertex right) => !left.Equals(right); + + public override bool Equals([NotNullWhen(true)] object obj) => obj is EdgeVertex other && Equals(other); + public bool Equals(EdgeVertex other) => Id == other.Id && Silo == other.Silo && IsMigratable == other.IsMigratable; + + public override int GetHashCode() => HashCode.Combine(Id, Silo, IsMigratable); +} + +/// +/// A candidate vertex to be transferred to another silo. +/// +[GenerateSerializer, DebuggerDisplay("Id = {Id} | Accumulated = {AccumulatedTransferScore}")] +internal sealed class CandidateVertex +{ + /// + /// The id of the candidate grain. + /// + [Id(0), Immutable] + public GrainId Id { get; init; } + + /// + /// The cost reduction expected from migrating the vertex with to another silo. + /// + [Id(1)] + public long AccumulatedTransferScore { get; set; } + + /// + /// These are all the vertices connected to the vertex with . + /// + /// These will be important when this vertex is removed from the max-sorted heap on the receiver silo. + [Id(2), Immutable] + public ImmutableArray ConnectedVertices { get; init; } = []; +} + +[GenerateSerializer, Immutable] +public readonly struct CandidateConnectedVertex(GrainId id, long transferScore) +{ + public GrainId Id { get; } = id; + public long TransferScore { get; } = transferScore; + + public static bool operator ==(CandidateConnectedVertex left, CandidateConnectedVertex right) => left.Equals(right); + public static bool operator !=(CandidateConnectedVertex left, CandidateConnectedVertex right) => !left.Equals(right); + + public override bool Equals([NotNullWhen(true)] object obj) => obj is CandidateConnectedVertex other && Equals(other); + public bool Equals(CandidateConnectedVertex other) => Id == other.Id && TransferScore == other.TransferScore; + + public override int GetHashCode() => HashCode.Combine(Id, TransferScore); +} + +[GenerateSerializer, Immutable] +internal sealed class AcceptExchangeRequest(SiloAddress sendingSilo, ImmutableArray exchangeSet, int activationCountSnapshot) +{ + [Id(0)] + public SiloAddress SendingSilo { get; } = sendingSilo; + + [Id(1)] + public ImmutableArray ExchangeSet { get; } = exchangeSet; + + [Id(2)] + public int ActivationCountSnapshot { get; } = activationCountSnapshot; +} + +[GenerateSerializer, Immutable] +internal sealed class AcceptExchangeResponse(AcceptExchangeResponse.ResponseType type, ImmutableArray exchangeSet) +{ + public static readonly AcceptExchangeResponse CachedExchangedRecently = new(ResponseType.ExchangedRecently, []); + public static readonly AcceptExchangeResponse CachedMutualExchangeAttempt = new(ResponseType.MutualExchangeAttempt, []); + + [Id(0)] + public ResponseType Type { get; } = type; + + [Id(1)] + public ImmutableArray ExchangeSet { get; } = exchangeSet; + + [GenerateSerializer] + public enum ResponseType + { + /// + /// The exchange was accepted and an exchange set is returned. + /// + Success = 0, + + /// + /// The other silo has been recently involved in another exchange. + /// + ExchangedRecently = 1, + + /// + /// An attempt to do an exchange between this and the other silo was about to happen at the same time. + /// + MutualExchangeAttempt = 2 + } +} diff --git a/src/Orleans.Core/Placement/Rebalancing/IImbalanceToleranceRule.cs b/src/Orleans.Core/Placement/Rebalancing/IImbalanceToleranceRule.cs new file mode 100644 index 0000000000..195d8e2821 --- /dev/null +++ b/src/Orleans.Core/Placement/Rebalancing/IImbalanceToleranceRule.cs @@ -0,0 +1,13 @@ +namespace Orleans.Placement.Rebalancing; + +/// +/// Represents a rule that controls the degree of imbalance between the number of grain activations (that is considered tolerable), when any pair of silos are exchanging activations. +/// +public interface IImbalanceToleranceRule +{ + /// + /// Checks if this rule is satisfied by . + /// + /// The imbalance between the exchanging silo pair that will be, if this method were to return + bool IsSatisfiedBy(uint imbalance); +} \ No newline at end of file diff --git a/src/Orleans.Core/Placement/Rebalancing/IMessageStatisticsSink.cs b/src/Orleans.Core/Placement/Rebalancing/IMessageStatisticsSink.cs new file mode 100644 index 0000000000..277d1287dd --- /dev/null +++ b/src/Orleans.Core/Placement/Rebalancing/IMessageStatisticsSink.cs @@ -0,0 +1,13 @@ +using Orleans.Runtime; + +namespace Orleans.Placement.Rebalancing; + +internal interface IMessageStatisticsSink +{ + void RecordMessage(Message message); +} + +internal sealed class NoOpMessageStatisticsSink : IMessageStatisticsSink +{ + public void RecordMessage(Message message) { } +} \ No newline at end of file diff --git a/src/Orleans.Core/Runtime/Constants.cs b/src/Orleans.Core/Runtime/Constants.cs index fbba2f9bc6..9c1d6dad4d 100644 --- a/src/Orleans.Core/Runtime/Constants.cs +++ b/src/Orleans.Core/Runtime/Constants.cs @@ -25,6 +25,7 @@ internal static class Constants public static readonly GrainType StreamPullingAgentType = SystemTargetGrainId.CreateGrainType("stream.agent"); public static readonly GrainType ManifestProviderType = SystemTargetGrainId.CreateGrainType("manifest"); public static readonly GrainType ActivationMigratorType = SystemTargetGrainId.CreateGrainType("migrator"); + public static readonly GrainType ActivationRebalancerType = SystemTargetGrainId.CreateGrainType("rebalancer"); public static readonly GrainId SiloDirectConnectionId = GrainId.Create( GrainType.Create(GrainTypePrefix.SystemPrefix + "silo"), diff --git a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs index e96b61a7c6..da33143fd8 100644 --- a/src/Orleans.Runtime/Activation/IGrainContextActivator.cs +++ b/src/Orleans.Runtime/Activation/IGrainContextActivator.cs @@ -5,6 +5,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Reflection; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Orleans.Concurrency; @@ -168,63 +169,23 @@ public class GrainTypeSharedContextResolver private readonly ConcurrentDictionary _components = new(); private readonly IConfigureGrainTypeComponents[] _configurators; private readonly GrainPropertiesResolver _grainPropertiesResolver; - private readonly GrainReferenceActivator _grainReferenceActivator; private readonly Func _createFunc; - private readonly IClusterManifestProvider _clusterManifestProvider; - private readonly GrainClassMap _grainClassMap; - private readonly IOptions _messagingOptions; - private readonly IOptions _collectionOptions; - private readonly IOptions _schedulingOptions; - private readonly PlacementStrategyResolver _placementStrategyResolver; - private readonly IGrainRuntime _grainRuntime; - private readonly ILogger _logger; private readonly IServiceProvider _serviceProvider; - private readonly SerializerSessionPool _serializerSessionPool; /// /// Initializes a new instance of the class. /// /// The grain type component configuration providers. /// The grain properties resolver. - /// The grain reference activator. - /// The cluster manifest provider. - /// The grain class map. - /// The grain placement strategy resolver. - /// The messaging options. - /// The grain activation collection options - /// The scheduling options - /// The grain runtime. - /// The logger. /// The service provider. - /// The serializer session pool. public GrainTypeSharedContextResolver( IEnumerable configurators, GrainPropertiesResolver grainPropertiesResolver, - GrainReferenceActivator grainReferenceActivator, - IClusterManifestProvider clusterManifestProvider, - GrainClassMap grainClassMap, - PlacementStrategyResolver placementStrategyResolver, - IOptions messagingOptions, - IOptions collectionOptions, - IOptions schedulingOptions, - IGrainRuntime grainRuntime, - ILogger logger, - IServiceProvider serviceProvider, - SerializerSessionPool serializerSessionPool) + IServiceProvider serviceProvider) { _configurators = configurators.ToArray(); _grainPropertiesResolver = grainPropertiesResolver; - _grainReferenceActivator = grainReferenceActivator; - _clusterManifestProvider = clusterManifestProvider; - _grainClassMap = grainClassMap; - _placementStrategyResolver = placementStrategyResolver; - _messagingOptions = messagingOptions; - _collectionOptions = collectionOptions; - _schedulingOptions = schedulingOptions; - _grainRuntime = grainRuntime; - _logger = logger; _serviceProvider = serviceProvider; - _serializerSessionPool = serializerSessionPool; _createFunc = Create; } @@ -237,19 +198,7 @@ public class GrainTypeSharedContextResolver private GrainTypeSharedContext Create(GrainType grainType) { - var result = new GrainTypeSharedContext( - grainType, - _clusterManifestProvider, - _grainClassMap, - _placementStrategyResolver, - _messagingOptions, - _collectionOptions, - _schedulingOptions, - _grainRuntime, - _logger, - _grainReferenceActivator, - _serviceProvider, - _serializerSessionPool); + var result = ActivatorUtilities.CreateInstance(_serviceProvider, grainType); var properties = _grainPropertiesResolver.GetGrainProperties(grainType); foreach (var configurator in _configurators) { diff --git a/src/Orleans.Runtime/Catalog/ActivationCollector.cs b/src/Orleans.Runtime/Catalog/ActivationCollector.cs index ea8f7e8c55..8d7607f715 100644 --- a/src/Orleans.Runtime/Catalog/ActivationCollector.cs +++ b/src/Orleans.Runtime/Catalog/ActivationCollector.cs @@ -512,7 +512,7 @@ private async Task DeactivateActivationsFromCollector(List mtcs.SetOneResult(); var reason = GetDeactivationReason(); diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 4947637a71..89ceb741cc 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -1738,19 +1738,19 @@ private async Task FinishDeactivating(CancellationToken cancellationToken) if (IsStuckDeactivating) { - CatalogInstruments.ActiviationShutdownViaDeactivateStuckActivation(); + CatalogInstruments.ActivationShutdownViaDeactivateStuckActivation(); } else if (migrated) { - CatalogInstruments.ActiviationShutdownViaMigration(); + CatalogInstruments.ActivationShutdownViaMigration(); } else if (_isInWorkingSet) { - CatalogInstruments.ActiviationShutdownViaDeactivateOnIdle(); + CatalogInstruments.ActivationShutdownViaApplication(); } else { - CatalogInstruments.ActiviationShutdownViaCollection(); + CatalogInstruments.ActivationShutdownViaCollection(); } _shared.InternalRuntime.ActivationWorkingSet.OnDeactivated(this); @@ -1761,7 +1761,7 @@ private async Task FinishDeactivating(CancellationToken cancellationToken) } catch (Exception exception) { - _shared.Logger.LogWarning(exception, "Exception disposing activation {Activation}", (ActivationData)this); + _shared.Logger.LogWarning(exception, "Exception disposing activation {Activation}", this); } UnregisterMessageTarget(); diff --git a/src/Orleans.Runtime/Catalog/Catalog.cs b/src/Orleans.Runtime/Catalog/Catalog.cs index d870a46dbc..6558349335 100644 --- a/src/Orleans.Runtime/Catalog/Catalog.cs +++ b/src/Orleans.Runtime/Catalog/Catalog.cs @@ -64,16 +64,14 @@ internal class Catalog : SystemTarget, ICatalog MessagingProcessingInstruments.RegisterActivationDataAllObserve(() => { long counter = 0; - lock (activations) + foreach (var activation in activations) { - foreach (var activation in activations) + if (activation.Value is ActivationData data) { - if (activation.Value is ActivationData data) - { - counter += data.GetRequestCount(); - } + counter += data.GetRequestCount(); } } + return counter; }); grainDirectory.SetSiloRemovedCatalogCallback(this.OnSiloStatusChange); diff --git a/src/Orleans.Runtime/Catalog/GrainTypeSharedContext.cs b/src/Orleans.Runtime/Catalog/GrainTypeSharedContext.cs index d7115c7787..f1097a5e3b 100644 --- a/src/Orleans.Runtime/Catalog/GrainTypeSharedContext.cs +++ b/src/Orleans.Runtime/Catalog/GrainTypeSharedContext.cs @@ -12,198 +12,197 @@ using Orleans.Serialization.Session; using Orleans.Serialization.TypeSystem; -namespace Orleans.Runtime +namespace Orleans.Runtime; + +/// +/// Functionality which is shared between all instances of a grain type. +/// +public class GrainTypeSharedContext { - /// - /// Functionality which is shared between all instances of a grain type. - /// - public class GrainTypeSharedContext + private readonly IServiceProvider _serviceProvider; + private readonly Dictionary _components = new(); + private InternalGrainRuntime? _internalGrainRuntime; + + public GrainTypeSharedContext( + GrainType grainType, + IClusterManifestProvider clusterManifestProvider, + GrainClassMap grainClassMap, + PlacementStrategyResolver placementStrategyResolver, + IOptions messagingOptions, + IOptions collectionOptions, + IOptions schedulingOptions, + IGrainRuntime grainRuntime, + ILoggerFactory loggerFactory, + GrainReferenceActivator grainReferenceActivator, + IServiceProvider serviceProvider, + SerializerSessionPool serializerSessionPool) { - private readonly IServiceProvider _serviceProvider; - private readonly Dictionary _components = new(); - private InternalGrainRuntime? _internalGrainRuntime; - - public GrainTypeSharedContext( - GrainType grainType, - IClusterManifestProvider clusterManifestProvider, - GrainClassMap grainClassMap, - PlacementStrategyResolver placementStrategyResolver, - IOptions messagingOptions, - IOptions collectionOptions, - IOptions schedulingOptions, - IGrainRuntime grainRuntime, - ILogger logger, - GrainReferenceActivator grainReferenceActivator, - IServiceProvider serviceProvider, - SerializerSessionPool serializerSessionPool) + if (!grainClassMap.TryGetGrainClass(grainType, out var grainClass)) { - if (!grainClassMap.TryGetGrainClass(grainType, out var grainClass)) - { - throw new KeyNotFoundException($"Could not find corresponding grain class for grain of type {grainType.ToString()}"); - } - - SerializerSessionPool = serializerSessionPool; - GrainTypeName = RuntimeTypeNameFormatter.Format(grainClass); - Logger = logger; - MessagingOptions = messagingOptions.Value; - GrainReferenceActivator = grainReferenceActivator; - _serviceProvider = serviceProvider; - MaxWarningRequestProcessingTime = messagingOptions.Value.ResponseTimeout.Multiply(5); - MaxRequestProcessingTime = messagingOptions.Value.MaxRequestProcessingTime; - PlacementStrategy = placementStrategyResolver.GetPlacementStrategy(grainType); - SchedulingOptions = schedulingOptions.Value; - Runtime = grainRuntime; - MigrationManager = _serviceProvider.GetService(); - - CollectionAgeLimit = GetCollectionAgeLimit( - grainType, - grainClass, - clusterManifestProvider.LocalGrainManifest, - collectionOptions.Value); + throw new KeyNotFoundException($"Could not find corresponding grain class for grain of type {grainType}"); } - /// - /// Gets the grain instance type name, if available. - /// - public string? GrainTypeName { get; } + SerializerSessionPool = serializerSessionPool; + GrainTypeName = RuntimeTypeNameFormatter.Format(grainClass); + Logger = loggerFactory.CreateLogger("Orleans.Grain"); + MessagingOptions = messagingOptions.Value; + GrainReferenceActivator = grainReferenceActivator; + _serviceProvider = serviceProvider; + MaxWarningRequestProcessingTime = messagingOptions.Value.ResponseTimeout.Multiply(5); + MaxRequestProcessingTime = messagingOptions.Value.MaxRequestProcessingTime; + PlacementStrategy = placementStrategyResolver.GetPlacementStrategy(grainType); + SchedulingOptions = schedulingOptions.Value; + Runtime = grainRuntime; + MigrationManager = _serviceProvider.GetService(); + + CollectionAgeLimit = GetCollectionAgeLimit( + grainType, + grainClass, + clusterManifestProvider.LocalGrainManifest, + collectionOptions.Value); + } + + /// + /// Gets the grain instance type name, if available. + /// + public string? GrainTypeName { get; } - private TimeSpan GetCollectionAgeLimit(GrainType grainType, Type grainClass, GrainManifest siloManifest, GrainCollectionOptions collectionOptions) + private static TimeSpan GetCollectionAgeLimit(GrainType grainType, Type grainClass, GrainManifest siloManifest, GrainCollectionOptions collectionOptions) + { + if (siloManifest.Grains.TryGetValue(grainType, out var properties) + && properties.Properties.TryGetValue(WellKnownGrainTypeProperties.IdleDeactivationPeriod, out var idleTimeoutString)) { - if (siloManifest.Grains.TryGetValue(grainType, out var properties) - && properties.Properties.TryGetValue(WellKnownGrainTypeProperties.IdleDeactivationPeriod, out var idleTimeoutString)) + if (string.Equals(idleTimeoutString, WellKnownGrainTypeProperties.IndefiniteIdleDeactivationPeriodValue)) { - if (string.Equals(idleTimeoutString, WellKnownGrainTypeProperties.IndefiniteIdleDeactivationPeriodValue)) - { - return Timeout.InfiniteTimeSpan; - } - - if (TimeSpan.TryParse(idleTimeoutString, out var result)) - { - return result; - } + return Timeout.InfiniteTimeSpan; } - if (collectionOptions.ClassSpecificCollectionAge.TryGetValue(grainClass.FullName!, out var specified)) + if (TimeSpan.TryParse(idleTimeoutString, out var result)) { - return specified; + return result; } - - return collectionOptions.CollectionAge; } - /// - /// Gets a component. - /// - /// The type specified in the corresponding call. - public TComponent? GetComponent() + if (collectionOptions.ClassSpecificCollectionAge.TryGetValue(grainClass.FullName!, out var specified)) { - if (typeof(TComponent) == typeof(PlacementStrategy) && PlacementStrategy is TComponent component) - { - return component; - } - - if (_components is null) return default; - _components.TryGetValue(typeof(TComponent), out var resultObj); - return (TComponent?)resultObj; + return specified; } - /// - /// Registers a component. - /// - /// The type which can be used as a key to . - public void SetComponent(TComponent? instance) - { - if (instance == null) - { - _components.Remove(typeof(TComponent)); - return; - } - - _components[typeof(TComponent)] = instance; - } + return collectionOptions.CollectionAge; + } - /// - /// Gets the duration after which idle grains are eligible for collection. - /// - public TimeSpan CollectionAgeLimit { get; } - - /// - /// Gets the logger. - /// - public ILogger Logger { get; } - - /// - /// Gets the serializer session pool. - /// - public SerializerSessionPool SerializerSessionPool { get; } - - /// - /// Gets the silo messaging options. - /// - public SiloMessagingOptions MessagingOptions { get; } - - /// - /// Gets the grain reference activator. - /// - public GrainReferenceActivator GrainReferenceActivator { get; } - - /// - /// Gets the maximum amount of time we expect a request to continue processing before it is considered hung. - /// - public TimeSpan MaxRequestProcessingTime { get; } - - /// - /// Gets the maximum amount of time we expect a request to continue processing before a warning may be logged. - /// - public TimeSpan MaxWarningRequestProcessingTime { get; } - - /// - /// Gets the placement strategy used by grains of this type. - /// - public PlacementStrategy PlacementStrategy { get; } - - /// - /// Gets the scheduling options. - /// - public SchedulingOptions SchedulingOptions { get; } - - /// - /// Gets the grain runtime. - /// - public IGrainRuntime Runtime { get; } - - /// - /// Gets the local activation migration manager. - /// - internal IActivationMigrationManager? MigrationManager { get; } - - /// - /// Gets the internal grain runtime. - /// - internal InternalGrainRuntime InternalRuntime => _internalGrainRuntime ??= _serviceProvider.GetRequiredService(); - - /// - /// Called on creation of an activation. - /// - /// The grain activation. - public void OnCreateActivation(IGrainContext grainContext) + /// + /// Gets a component. + /// + /// The type specified in the corresponding call. + public TComponent? GetComponent() + { + if (typeof(TComponent) == typeof(PlacementStrategy) && PlacementStrategy is TComponent component) { - GrainInstruments.IncrementGrainCounts(GrainTypeName); + return component; } - /// - /// Called when an activation is disposed. - /// - /// The grain activation. - public void OnDestroyActivation(IGrainContext grainContext) + if (_components is null) return default; + _components.TryGetValue(typeof(TComponent), out var resultObj); + return (TComponent?)resultObj; + } + + /// + /// Registers a component. + /// + /// The type which can be used as a key to . + public void SetComponent(TComponent? instance) + { + if (instance == null) { - GrainInstruments.DecrementGrainCounts(GrainTypeName); + _components.Remove(typeof(TComponent)); + return; } + + _components[typeof(TComponent)] = instance; + } + + /// + /// Gets the duration after which idle grains are eligible for collection. + /// + public TimeSpan CollectionAgeLimit { get; } + + /// + /// Gets the logger. + /// + public ILogger Logger { get; } + + /// + /// Gets the serializer session pool. + /// + public SerializerSessionPool SerializerSessionPool { get; } + + /// + /// Gets the silo messaging options. + /// + public SiloMessagingOptions MessagingOptions { get; } + + /// + /// Gets the grain reference activator. + /// + public GrainReferenceActivator GrainReferenceActivator { get; } + + /// + /// Gets the maximum amount of time we expect a request to continue processing before it is considered hung. + /// + public TimeSpan MaxRequestProcessingTime { get; } + + /// + /// Gets the maximum amount of time we expect a request to continue processing before a warning may be logged. + /// + public TimeSpan MaxWarningRequestProcessingTime { get; } + + /// + /// Gets the placement strategy used by grains of this type. + /// + public PlacementStrategy PlacementStrategy { get; } + + /// + /// Gets the scheduling options. + /// + public SchedulingOptions SchedulingOptions { get; } + + /// + /// Gets the grain runtime. + /// + public IGrainRuntime Runtime { get; } + + /// + /// Gets the local activation migration manager. + /// + internal IActivationMigrationManager? MigrationManager { get; } + + /// + /// Gets the internal grain runtime. + /// + internal InternalGrainRuntime InternalRuntime => _internalGrainRuntime ??= _serviceProvider.GetRequiredService(); + + /// + /// Called on creation of an activation. + /// + /// The grain activation. + public void OnCreateActivation(IGrainContext grainContext) + { + GrainInstruments.IncrementGrainCounts(grainContext.GrainId.Type.ToString()); } - internal interface IActivationLifecycleObserver + /// + /// Called when an activation is disposed. + /// + /// The grain activation. + public void OnDestroyActivation(IGrainContext grainContext) { - void OnCreateActivation(IGrainContext grainContext); - void OnDestroyActivation(IGrainContext grainContext); + GrainInstruments.DecrementGrainCounts(grainContext.GrainId.Type.ToString()); } } + +internal interface IActivationLifecycleObserver +{ + void OnCreateActivation(IGrainContext grainContext); + void OnDestroyActivation(IGrainContext grainContext); +} diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs new file mode 100644 index 0000000000..3c3596a129 --- /dev/null +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -0,0 +1,150 @@ +using System; +using Microsoft.Extensions.Options; +using Orleans.Runtime; + +namespace Orleans.Configuration; + +public sealed class ActiveRebalancingOptions +{ + /// + /// + /// The maximum number of edges to retain in-memory during a rebalancing cycle. An edge represents how many calls were made from one grain to another. + /// + /// + /// If this number is N, it does not mean that N activations will be migrated after a rebalancing cycle. + /// It also does not mean that if any activation ranked very high, that it will rank high at the next cycle. + /// At the most extreme case, the number of activations that will be migrated, will equal this number, so this should give you some idea as to setting a reasonable value for this. + /// + /// + /// + /// In order to preserve memory, the most heaviest links are recorded in a probabilistic way, so there is an inherent error associated with that. + /// That error is inversely proportional to this value, so values under 100 are not recommended. If you notice that the system is not converging fast enough, do consider increasing this number. + /// + public uint MaxEdgeCount { get; set; } = + + + + 10 * + + + + DEFAULT_MAX_EDGE_COUNT; + + /// + /// The default value of . + /// + public const uint DEFAULT_MAX_EDGE_COUNT = 10_000; + + /// + /// The minimum time given to this silo to gather statistics before triggering the first rebalancing cycle. + /// + /// The actual due time is picked randomly between this and . + public TimeSpan MinRebalancingDueTime { get; set; } = DEFAULT_MINUMUM_REBALANCING_DUE_TIME; + + /// + /// The default value of . + /// + public static readonly TimeSpan DEFAULT_MINUMUM_REBALANCING_DUE_TIME = TimeSpan.FromMinutes(1); + + /// + /// The maximum time given to this silo to gather statistics before triggering the first rebalancing cycle. + /// + /// + /// The actual due time is picked randomly between this and . + /// For optimal results, you should aim to give this an extra 10 seconds x the maximum anticipated silo count in the cluster. + /// + public TimeSpan MaxRebalancingDueTime { get; set; } = DEFAULT_MAXIMUM_REBALANCING_DUE_TIME; + + /// + /// The default value of . + /// + public static readonly TimeSpan DEFAULT_MAXIMUM_REBALANCING_DUE_TIME = TimeSpan.FromMinutes(2); + + /// + /// The cycle upon which this silo will trigger a rebalancing session with another silo. + /// + /// Must be greater than , you should aim for at least 2 times that of . + public TimeSpan RebalancingPeriod { get; set; } = DEFAULT_REBALANCING_PERIOD; + + /// + /// The default value of . + /// + public static readonly TimeSpan DEFAULT_REBALANCING_PERIOD = TimeSpan.FromMinutes(2); + + /// + /// The minimum time needed for a silo to recover from a previous rebalancing. + /// Until this time has elapsed, this silo will not take part in any rebalancing attempt from another silo. + /// + /// + /// + /// While this silo will refuse rebalancing attempts from other silos, if falls within this period, than + /// this silo will attempt a rebalancing with another silo, but this silo will be the initiator, not the other way around. + /// + /// Must be less than , you should aim for at least 1/2 times that of . + /// + public TimeSpan RecoveryPeriod { get; set; } = DEFAULT_RECOVERY_PERIOD; + + /// + /// The default value of . + /// + public static readonly TimeSpan DEFAULT_RECOVERY_PERIOD = TimeSpan.FromMinutes(1); + + /// + /// The maximum number of unprocessed edges to buffer. If this number is exceeded, the oldest edges will be discarded. + /// + public int MaxUnprocessedEdges { get; set; } = DEFAULT_MAX_UNPROCESSED_EDGES; + + /// + /// The default value of . + /// + public const int DEFAULT_MAX_UNPROCESSED_EDGES = 100_000; +} + +internal sealed class ActiveRebalancingOptionsValidator(IOptions options) : IConfigurationValidator +{ + private readonly ActiveRebalancingOptions _options = options.Value; + + public void ValidateConfiguration() + { + if (_options.MaxEdgeCount == 0) + { + ThrowMustBeGreaterThanZero(nameof(ActiveRebalancingOptions.MaxEdgeCount)); + } + + if (_options.MinRebalancingDueTime == TimeSpan.Zero) + { + ThrowMustBeGreaterThanZero(nameof(ActiveRebalancingOptions.MinRebalancingDueTime)); + } + + if (_options.MaxRebalancingDueTime == TimeSpan.Zero) + { + ThrowMustBeGreaterThanZero(nameof(ActiveRebalancingOptions.MaxRebalancingDueTime)); + } + + if (_options.RebalancingPeriod == TimeSpan.Zero) + { + ThrowMustBeGreaterThanZero(nameof(ActiveRebalancingOptions.RebalancingPeriod)); + } + + if (_options.RecoveryPeriod == TimeSpan.Zero) + { + ThrowMustBeGreaterThanZero(nameof(ActiveRebalancingOptions.RecoveryPeriod)); + } + + if (_options.MaxRebalancingDueTime <= _options.MinRebalancingDueTime) + { + ThrowMustBeGreaterThan(nameof(ActiveRebalancingOptions.MaxRebalancingDueTime), nameof(ActiveRebalancingOptions.MinRebalancingDueTime)); + } + + if (_options.RebalancingPeriod <= _options.RecoveryPeriod) + { + ThrowMustBeGreaterThan(nameof(ActiveRebalancingOptions.RebalancingPeriod), nameof(ActiveRebalancingOptions.RecoveryPeriod)); + } + } + + private static void ThrowMustBeGreaterThanZero(string propertyName) + => throw new OrleansConfigurationException($"{propertyName} must be greater than 0"); + + private static void ThrowMustBeGreaterThan(string name1, string name2) + => throw new OrleansConfigurationException($"{name1} must be greater than {name2}"); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs b/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs index 7c51838835..ca97e15884 100644 --- a/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs @@ -12,7 +12,7 @@ public class SiloMessagingOptions : MessagingOptions /// /// . /// - private TimeSpan systemResponseTimeout = TimeSpan.FromSeconds(30); + private TimeSpan systemResponseTimeout = TimeSpan.FromSeconds(300); /// /// Gets or sets the number of parallel queues and attendant threads used by the silo to send outbound diff --git a/src/Orleans.Runtime/Hosting/ActiveRebalancingExtensions.cs b/src/Orleans.Runtime/Hosting/ActiveRebalancingExtensions.cs new file mode 100644 index 0000000000..7c5e023f06 --- /dev/null +++ b/src/Orleans.Runtime/Hosting/ActiveRebalancingExtensions.cs @@ -0,0 +1,51 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration.Internal; +using Orleans.Placement.Rebalancing; +using Orleans.Runtime; +using Orleans.Configuration; +using Orleans.Runtime.Placement.Rebalancing; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Hosting; + +#nullable enable +public static class ActiveRebalancingExtensions +{ + /// + /// Adds support for active-rebalancing in this silo. + /// + [Experimental("ORLEANSEXP001")] + public static ISiloBuilder AddActiveRebalancing(this ISiloBuilder builder) + => builder.AddActiveRebalancing(); + + /// + /// Adds support for active-rebalancing in this silo. + /// + [Experimental("ORLEANSEXP001")] + public static ISiloBuilder AddActiveRebalancing(this ISiloBuilder builder) where TRule : class, IImbalanceToleranceRule + => builder + .ConfigureServices(services => services.AddActiveRebalancing()); + + private static IServiceCollection AddActiveRebalancing(this IServiceCollection services) where TRule : class, IImbalanceToleranceRule + { + services.AddTransient(); + + if (typeof(TRule) == typeof(DefaultImbalanceRule)) + { + services.AddSingleton(); + services.AddFromExisting(); + services.AddFromExisting, DefaultImbalanceRule>(); + } + else + { + services.AddSingleton(); + } + + services.AddSingleton(); + services.AddSingleton(); + services.AddFromExisting(); + services.AddFromExisting, ActivationRebalancer>(); + + return services; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index f814a109db..83a8f87e87 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -42,6 +42,7 @@ using Microsoft.Extensions.Configuration; using Orleans.Serialization.Internal; using Orleans.Core; +using Orleans.Placement.Rebalancing; namespace Orleans.Hosting { @@ -359,6 +360,7 @@ internal static void AddDefaultServices(ISiloBuilder builder) (sp, _) => sp.GetRequiredService()); // Networking + services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); diff --git a/src/Orleans.Runtime/MembershipService/ISiloStatusOracle.cs b/src/Orleans.Runtime/MembershipService/ISiloStatusOracle.cs index 5babdbb7dd..dccaa7b321 100644 --- a/src/Orleans.Runtime/MembershipService/ISiloStatusOracle.cs +++ b/src/Orleans.Runtime/MembershipService/ISiloStatusOracle.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Collections.Immutable; namespace Orleans.Runtime { @@ -22,6 +23,11 @@ public interface ISiloStatusOracle /// SiloAddress SiloAddress { get; } + /// + /// Gets the currently active silos. + /// + ImmutableArray GetActiveSilos(); + /// /// Gets the status of a given silo. /// This method returns an approximate view on the status of a given silo. diff --git a/src/Orleans.Runtime/MembershipService/SiloStatusOracle.cs b/src/Orleans.Runtime/MembershipService/SiloStatusOracle.cs index c37471f69b..2700d6f52d 100644 --- a/src/Orleans.Runtime/MembershipService/SiloStatusOracle.cs +++ b/src/Orleans.Runtime/MembershipService/SiloStatusOracle.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using Microsoft.Extensions.Logging; using System.Threading; +using System.Collections.Immutable; namespace Orleans.Runtime.MembershipService { @@ -14,6 +15,7 @@ internal class SiloStatusOracle : ISiloStatusOracle private MembershipTableSnapshot cachedSnapshot; private Dictionary siloStatusCache = new Dictionary(); private Dictionary siloStatusCacheOnlyActive = new Dictionary(); + private ImmutableArray _activeSilos = []; public SiloStatusOracle( ILocalSiloDetails localSiloDetails, @@ -49,35 +51,53 @@ public SiloStatus GetApproximateSiloStatus(SiloAddress silo) return status; } + public ImmutableArray GetActiveSilos() + { + EnsureFreshCache(); + return _activeSilos; + } + public Dictionary GetApproximateSiloStatuses(bool onlyActive = false) { - if (ReferenceEquals(this.cachedSnapshot, this.membershipTableManager.MembershipTableSnapshot)) + EnsureFreshCache(); + return onlyActive ? this.siloStatusCacheOnlyActive : this.siloStatusCache; + } + + private void EnsureFreshCache() + { + var currentMembership = this.membershipTableManager.MembershipTableSnapshot; + if (ReferenceEquals(this.cachedSnapshot, currentMembership)) { - return onlyActive ? this.siloStatusCacheOnlyActive : this.siloStatusCache; + return; } lock (this.cacheUpdateLock) { - var currentMembership = this.membershipTableManager.MembershipTableSnapshot; + currentMembership = this.membershipTableManager.MembershipTableSnapshot; if (ReferenceEquals(this.cachedSnapshot, currentMembership)) { - return onlyActive ? this.siloStatusCacheOnlyActive : this.siloStatusCache; + return; } var newSiloStatusCache = new Dictionary(); var newSiloStatusCacheOnlyActive = new Dictionary(); + var newActiveSilos = ImmutableArray.CreateBuilder(); foreach (var entry in currentMembership.Entries) { var silo = entry.Key; var status = entry.Value.Status; newSiloStatusCache[silo] = status; - if (status == SiloStatus.Active) newSiloStatusCacheOnlyActive[silo] = status; + if (status == SiloStatus.Active) + { + newSiloStatusCacheOnlyActive[silo] = status; + newActiveSilos.Add(silo); + } } Interlocked.Exchange(ref this.cachedSnapshot, currentMembership); this.siloStatusCache = newSiloStatusCache; this.siloStatusCacheOnlyActive = newSiloStatusCacheOnlyActive; - return onlyActive ? newSiloStatusCacheOnlyActive : newSiloStatusCache; + _activeSilos = newActiveSilos.ToImmutable(); } } diff --git a/src/Orleans.Runtime/Messaging/MessageCenter.cs b/src/Orleans.Runtime/Messaging/MessageCenter.cs index 96564a950f..4dc817f01f 100644 --- a/src/Orleans.Runtime/Messaging/MessageCenter.cs +++ b/src/Orleans.Runtime/Messaging/MessageCenter.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Orleans.Configuration; +using Orleans.Placement.Rebalancing; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.Placement; using Orleans.Serialization.Invocation; @@ -21,6 +22,7 @@ internal class MessageCenter : IMessageCenter, IAsyncDisposable private readonly SiloMessagingOptions messagingOptions; private readonly PlacementService placementService; private readonly GrainLocator _grainLocator; + private readonly IMessageStatisticsSink _messageStatisticsSink; private readonly ILogger log; private readonly Catalog catalog; private bool stopped; @@ -38,7 +40,8 @@ internal class MessageCenter : IMessageCenter, IAsyncDisposable RuntimeMessagingTrace messagingTrace, IOptions messagingOptions, PlacementService placementService, - GrainLocator grainLocator) + GrainLocator grainLocator, + IMessageStatisticsSink messageStatisticsSink) { this.catalog = catalog; this.messagingOptions = messagingOptions.Value; @@ -47,6 +50,7 @@ internal class MessageCenter : IMessageCenter, IAsyncDisposable this.messagingTrace = messagingTrace; this.placementService = placementService; _grainLocator = grainLocator; + _messageStatisticsSink = messageStatisticsSink; this.log = logger; this.messageFactory = messageFactory; this._siloAddress = siloDetails.SiloAddress; @@ -66,8 +70,14 @@ internal class MessageCenter : IMessageCenter, IAsyncDisposable public bool TryDeliverToProxy(Message msg) { if (!msg.TargetGrain.IsClient()) return false; - if (this.Gateway is Gateway gateway && gateway.TryDeliverToProxy(msg)) return true; - return this.hostedClient is HostedClient client && client.TryDispatchToClient(msg); + if (this.Gateway is Gateway gateway && gateway.TryDeliverToProxy(msg) + || this.hostedClient is HostedClient client && client.TryDispatchToClient(msg)) + { + _messageStatisticsSink.RecordMessage(msg); + return true; + } + + return false; } public async Task StopAsync() @@ -173,6 +183,7 @@ public void SendMessage(Message msg) } messagingTrace.OnSendMessage(msg); + if (targetSilo.Matches(_siloAddress)) { if (log.IsEnabled(LogLevel.Trace)) @@ -532,6 +543,7 @@ public void ReceiveMessage(Message msg) } targetActivation.ReceiveMessage(msg); + _messageStatisticsSink.RecordMessage(msg); } } catch (Exception ex) diff --git a/src/Orleans.Runtime/Networking/GatewayInboundConnection.cs b/src/Orleans.Runtime/Networking/GatewayInboundConnection.cs index b3e62b279b..b1ecd0b8d8 100644 --- a/src/Orleans.Runtime/Networking/GatewayInboundConnection.cs +++ b/src/Orleans.Runtime/Networking/GatewayInboundConnection.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Orleans.Configuration; using Orleans.Messaging; +using Orleans.Placement.Rebalancing; namespace Orleans.Runtime.Messaging { @@ -45,13 +46,13 @@ internal sealed class GatewayInboundConnection : Connection protected override void RecordMessageReceive(Message msg, int numTotalBytes, int headerBytes) { - MessagingInstruments.OnMessageReceive(msg, numTotalBytes, headerBytes, ConnectionDirection); + MessagingInstruments.OnMessageReceive(msg, numTotalBytes, headerBytes); GatewayInstruments.GatewayReceived.Add(1); } protected override void RecordMessageSend(Message msg, int numTotalBytes, int headerBytes) { - MessagingInstruments.OnMessageSend(msg, numTotalBytes, headerBytes, ConnectionDirection); + MessagingInstruments.OnMessageSend(msg, numTotalBytes, headerBytes); GatewayInstruments.GatewaySent.Add(1); } diff --git a/src/Orleans.Runtime/Networking/SiloConnection.cs b/src/Orleans.Runtime/Networking/SiloConnection.cs index 7580107287..04d8eb2137 100644 --- a/src/Orleans.Runtime/Networking/SiloConnection.cs +++ b/src/Orleans.Runtime/Networking/SiloConnection.cs @@ -9,6 +9,7 @@ using Microsoft.Extensions.Logging; using Orleans.Configuration; using Orleans.Messaging; +using Orleans.Placement.Rebalancing; using Orleans.Serialization.Invocation; namespace Orleans.Runtime.Messaging @@ -57,12 +58,12 @@ internal sealed class SiloConnection : Connection protected override void RecordMessageReceive(Message msg, int numTotalBytes, int headerBytes) { - MessagingInstruments.OnMessageReceive(msg, numTotalBytes, headerBytes, ConnectionDirection, RemoteSiloAddress); + MessagingInstruments.OnMessageReceive(msg, numTotalBytes, headerBytes); } protected override void RecordMessageSend(Message msg, int numTotalBytes, int headerBytes) { - MessagingInstruments.OnMessageSend(msg, numTotalBytes, headerBytes, ConnectionDirection, RemoteSiloAddress); + MessagingInstruments.OnMessageSend(msg, numTotalBytes, headerBytes); } protected override void OnReceivedMessage(Message msg) diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index 3428b7905b..9c8dee8154 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -68,7 +68,7 @@ internal class PlacementService : IPlacementContext /// public Task AddressMessage(Message message) { - if (message.IsFullyAddressed) return Task.CompletedTask; + if (message.IsTargetFullyAddressed) return Task.CompletedTask; if (message.TargetGrain.IsDefault) ThrowMissingAddress(); var grainId = message.TargetGrain; diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs new file mode 100644 index 0000000000..06d296b179 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs @@ -0,0 +1,40 @@ +using System; +using Microsoft.Extensions.Logging; + +namespace Orleans.Runtime.Placement.Rebalancing; + +internal partial class ActivationRebalancer +{ + [LoggerMessage(Level = LogLevel.Debug, Message = "I will periodically initiate the exchange protocol every {RebalancingPeriod} starting in {DueTime}.")] + private partial void LogPeriodicallyInvokeProtocol(TimeSpan rebalancingPeriod, TimeSpan dueTime); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Active rebalancing is enabled, but the cluster contains only one silo. Waiting for at least another silo to join the cluster to proceed.")] + private partial void LogSingleSiloCluster(); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Exchange set for candidate silo {CandidateSilo} is empty. I will try the next best candidate (if one is available), otherwise I will wait for my next period to come.")] + private partial void LogExchangeSetIsEmpty(SiloAddress candidateSilo); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Beginning exchange protocol between {ThisSilo} and {CandidateSilo}.")] + private partial void LogBeginningProtocol(SiloAddress thisSilo, SiloAddress candidateSilo); + + [LoggerMessage(Level = LogLevel.Debug, Message = "I got an exchange request from {SendingSilo}, but I have been recently involved in another exchange {LastExchangeDuration} ago. My recovery period is {RecoveryPeriod}")] + private partial void LogExchangedRecently(SiloAddress sendingSilo, TimeSpan lastExchangeDuration, TimeSpan recoveryPeriod); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Exchange request from {ThisSilo} failed, due to {CandidateSilo} having been recently involved in another exchange. I will try the next best candidate (if one is available), otherwise I will wait for my next period to come.")] + private partial void LogExchangedRecentlyResponse(SiloAddress thisSilo, SiloAddress candidateSilo); + + [LoggerMessage(Level = LogLevel.Debug, Message = "I got an exchange request from {SendingSilo}, but I am performing one with it at the same time. I have phase-shifted my timer to avoid these conflicts.")] + private partial void LogMutualExchangeAttempt(SiloAddress sendingSilo); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Exchange request from {ThisSilo} superseded by a mutual exchange attempt with {CandidateSilo}.")] + private partial void LogMutualExchangeAttemptResponse(SiloAddress thisSilo, SiloAddress candidateSilo); + + [LoggerMessage(Level = LogLevel.Debug, Message = "I have successfully finalized my part of the exchange protocol. It was decided that I will take on a total of {ActivationCount} activations.")] + private partial void LogProtocolFinalized(int activationCount); + + [LoggerMessage(Level = LogLevel.Warning, Message = "An error occurred while performing exchange request from {ThisSilo} to {CandidateSilo}. I will try the next best candidate (if one is available), otherwise I will wait for my next period to come.")] + private partial void LogErrorOnProtocolExecution(Exception exception, SiloAddress thisSilo, SiloAddress candidateSilo); + + [LoggerMessage(Level = LogLevel.Warning, Message = "There was an issue during the migration of the activation set initiated by {ThisSilo}.")] + private partial void LogErrorOnMigratingActivations(Exception exception, SiloAddress thisSilo); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs new file mode 100644 index 0000000000..c8e5c8ee8f --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -0,0 +1,90 @@ +#nullable enable +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Placement.Rebalancing; +using Orleans.Runtime.Internal; + +namespace Orleans.Runtime.Placement.Rebalancing; + +internal partial class ActivationRebalancer : IMessageStatisticsSink +{ + private readonly CancellationTokenSource _shutdownCts = new(); + private Task? _processPendingEdgesTask; + + public void StartProcessingEdges() + { + using var _ = new ExecutionContextSuppressor(); + _processPendingEdgesTask = ProcessPendingEdges(_shutdownCts.Token); + + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace("{Service} has started.", nameof(ActivationRebalancer)); + } + } + + public async Task StopProcessingEdgesAsync(CancellationToken cancellationToken) + { + _shutdownCts.Cancel(); + if (_processPendingEdgesTask is null) + { + return; + } + + _pendingMessageEvent.Signal(); + await _processPendingEdgesTask.WaitAsync(cancellationToken).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace("{Service} has stopped.", nameof(ActivationRebalancer)); + } + } + + private async Task ProcessPendingEdges(CancellationToken cancellationToken) + { + const int MaxCyclesPerYield = 100; + await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding | ConfigureAwaitOptions.ContinueOnCapturedContext); + + var drainBuffer = new Message[256]; + var cyclesPerYield = 100; + while (!cancellationToken.IsCancellationRequested) + { + var count = _pendingMessages.DrainTo(drainBuffer); + if (count > 0) + { + foreach (var message in drainBuffer[..count]) + { + _messageFilter.IsAcceptable(message, out var isSenderMigratable, out var isTargetMigratable); + + Edge edge = new( + new(message.SendingGrain, message.SendingSilo, isSenderMigratable), + new(message.TargetGrain, message.TargetSilo, isTargetMigratable)); + _edgeWeights.Add(edge); + } + } + else + { + await _pendingMessageEvent.WaitAsync(); + } + + if (++cyclesPerYield >= MaxCyclesPerYield) + { + cyclesPerYield = 0; + await Task.Yield(); + } + } + } + + public void RecordMessage(Message message) + { + if (!_enableMessageSampling || !_messageFilter.IsAcceptable(message, out var isSenderMigratable, out var isTargetMigratable)) + { + return; + } + + if (_pendingMessages.TryAdd(message) == Utilities.BufferStatus.Success) + { + _pendingMessageEvent.Signal(); + } + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs new file mode 100644 index 0000000000..5c0fd1fe2d --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -0,0 +1,759 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using System.Runtime.CompilerServices; +using System.Diagnostics; +using System.Collections.Immutable; +using Orleans.Core.Internal; +using System.Data; +using Orleans.Placement.Rebalancing; +using System.Threading; +using Orleans.Internal; +using Orleans.Configuration; +using Orleans.Runtime.Utilities; + +namespace Orleans.Runtime.Placement.Rebalancing; + +// See: https://www.microsoft.com/en-us/research/wp-content/uploads/2016/06/eurosys16loca_camera_ready-1.pdf +internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRebalancerSystemTarget, ILifecycleParticipant, IDisposable, ISiloStatusListener +{ + private readonly ILogger _logger; + private readonly ISiloStatusOracle _siloStatusOracle; + private readonly IInternalGrainFactory _grainFactory; + private readonly IRebalancingMessageFilter _messageFilter; + private readonly IImbalanceToleranceRule _toleranceRule; + private readonly ActivationDirectory _activationDirectory; + private readonly ActiveRebalancingOptions _options; + private readonly StripedMpscBuffer _pendingMessages; + private readonly SingleWaiterAutoResetEvent _pendingMessageEvent = new() { RunContinuationsAsynchronously = true }; + private readonly FrequentEdgeCounter _edgeWeights; + private readonly IGrainTimer _timer; + private SiloAddress? _currentExchangeSilo; + private CoarseStopwatch _lastExchangedStopwatch; + private int _activationCountOffset; + private bool _enableMessageSampling; + + public ActivationRebalancer( + ISiloStatusOracle siloStatusOracle, + ILocalSiloDetails localSiloDetails, + ILoggerFactory loggerFactory, + IInternalGrainFactory internalGrainFactory, + IRebalancingMessageFilter messageFilter, + IImbalanceToleranceRule toleranceRule, + ActivationDirectory activationDirectory, + Catalog catalog, + IOptions options) + : base(Constants.ActivationRebalancerType, localSiloDetails.SiloAddress, loggerFactory) + { + _logger = loggerFactory.CreateLogger(); + _options = options.Value; + _siloStatusOracle = siloStatusOracle; + _grainFactory = internalGrainFactory; + _messageFilter = messageFilter; + _toleranceRule = toleranceRule; + _activationDirectory = activationDirectory; + _edgeWeights = new((int)options.Value.MaxEdgeCount); + _pendingMessages = new StripedMpscBuffer(Environment.ProcessorCount, options.Value.MaxUnprocessedEdges / Environment.ProcessorCount); + + _lastExchangedStopwatch = CoarseStopwatch.StartNew((long)options.Value.RecoveryPeriod.Add(TimeSpan.FromDays(2)).TotalMilliseconds); + catalog.RegisterSystemTarget(this); + _siloStatusOracle.SubscribeToSiloStatusEvents(this); + _timer = RegisterTimer(_ => TriggerExchangeRequest().AsTask(), null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + } + + private Task OnActiveStart(CancellationToken cancellationToken) + { + Scheduler.QueueAction(() => + { + var dueTime = RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime); + RegisterOrUpdateTimer(dueTime); + StartProcessingEdges(); + }); + + return Task.CompletedTask; + } + + public ValueTask ResetCounters() + { + _pendingMessages.Clear(); + _edgeWeights.Clear(); + return ValueTask.CompletedTask; + } + + ValueTask IActivationRebalancerSystemTarget.GetActivationCount() => new(_activationDirectory.Count); + ValueTask IActivationRebalancerSystemTarget.SetActivationCountOffset(int activationCountOffset) + { + _activationCountOffset = activationCountOffset; + return ValueTask.CompletedTask; + } + + private void RegisterOrUpdateTimer(TimeSpan dueTime) + { + _timer.Change(dueTime, dueTime); + LogPeriodicallyInvokeProtocol(_options.RebalancingPeriod, dueTime); + } + + public async ValueTask TriggerExchangeRequest() + { + var silos = _siloStatusOracle.GetActiveSilos(); + if (silos.Length == 1) + { + //_enableMessageSampling = false; + LogSingleSiloCluster(); + return; // If its a single-silo cluster we have no business doing any kind of rebalancing + } + else if (!_enableMessageSampling) + { +// _enableMessageSampling = true; + return; + } + + var sets = CreateCandidateSets(silos); + + var countWithNoExchangeSet = 0; + foreach (var set in sets) + { + if (_currentExchangeSilo is not null) + { + // Skip this round if we are already in the process of exchanging with another silo. + return; + } + + (var candidateSilo, var exchangeSet, var _) = set; + if (exchangeSet.Length == 0) + { + countWithNoExchangeSet++; + LogExchangeSetIsEmpty(candidateSilo); + continue; + } + + try + { + // Set the exchange partner for the duration of the operation. + // This prevents other requests from interleaving. + _currentExchangeSilo = candidateSilo; + + LogBeginningProtocol(Silo, candidateSilo); + var remoteRef = IActivationRebalancerSystemTarget.GetReference(_grainFactory, candidateSilo); + var sw2 = ValueStopwatch.StartNew(); + _logger.LogInformation("Sending AcceptExchangeRequest"); + var response = await remoteRef.AcceptExchangeRequest(new(Silo, exchangeSet, GetLocalActivationCount())); + _logger.LogInformation("Sent AcceptExchangeRequest. It took {Elapsed}", sw2.Elapsed); + + switch (response.Type) + { + case AcceptExchangeResponse.ResponseType.Success: + // Exchange was successful, no need to iterate over another candidate. + await FinalizeProtocol(response.ExchangeSet, exchangeSet.Select(x => x.Id).Union(response.ExchangeSet).ToImmutableArray(), isReceiver: false); + return; + case AcceptExchangeResponse.ResponseType.ExchangedRecently: + // The remote silo has been recently involved in another exchange, try the next best candidate. + LogExchangedRecentlyResponse(Silo, candidateSilo); + continue; + case AcceptExchangeResponse.ResponseType.MutualExchangeAttempt: + // The remote silo is exchanging with this silo already and the exchange the remote silo initiated + // took precedence over the one this silo is initiating. + LogMutualExchangeAttemptResponse(Silo, candidateSilo); + return; + } + } + catch (Exception ex) + { + LogErrorOnProtocolExecution(ex, Silo, candidateSilo); + continue; // there was some problem, try the next best candidate + } + finally + { + _currentExchangeSilo = null; + } + } + + /* + if (countWithNoExchangeSet == sets.Count) + { + // Disable message sampling for now, since there were no exchanges performed. + _logger.LogDebug("Placement has stabilized. Disabling sampling."); + _enableMessageSampling = false; + } + */ + } + + private int GetLocalActivationCount() => _activationDirectory.Count + _activationCountOffset; + + public async ValueTask AcceptExchangeRequest(AcceptExchangeRequest request) + { + if (request.SendingSilo.Equals(_currentExchangeSilo) && Silo.CompareTo(request.SendingSilo) <= 0) + { + // Reject the request, as we are already in the process of exchanging with the sending silo. + // The '<=' comparison here is used to break the tie in case both silos are exchanging with each other. + + // We pick some random time between 'min' and 'max' and than subtract from it 'min'. We do this so this silo doesn't have to wait for 'min + random', + // as it did the very first time this was started. It is guaranteed that 'random - min' >= 0; as 'random' will be at the least equal to 'min'. + RegisterOrUpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime) - _options.MinRebalancingDueTime); + LogMutualExchangeAttempt(request.SendingSilo); + + return AcceptExchangeResponse.CachedMutualExchangeAttempt; + } + + var lastExchangeElapsed = _lastExchangedStopwatch.Elapsed; + if (lastExchangeElapsed < _options.RecoveryPeriod) + { + LogExchangedRecently(request.SendingSilo, lastExchangeElapsed, _options.RecoveryPeriod); + return AcceptExchangeResponse.CachedExchangedRecently; + } + + // Set the exchange silo for the duration of the request. + // This prevents other requests from interleaving. + _currentExchangeSilo = request.SendingSilo; + + var sw = ValueStopwatch.StartNew(); + try + { + var remoteSet = request.ExchangeSet; + var localSet = CreateCandidateSet(CreateLocalVertexEdges(), request.SendingSilo); + + if (localSet.Count == 0) + { + // We have nothing to give back (very fringe case), so just accept the set. + var set = remoteSet.Select(x => x.Id).ToImmutableArray(); + await FinalizeProtocol(set, set, isReceiver: true); + + return new(AcceptExchangeResponse.ResponseType.Success, []); + } + + List<(GrainId Grain, SiloAddress Silo, long TransferScore)> toMigrate = []; + + // We need to determine 2 subsets: + // - One that originates from sending silo (request.ExchangeSet) and will be (partially) accepted from this silo. + // - One that originates from this silo (candidateSet) and will be (fully) accepted from the sending silo. + var remoteActivations = request.ActivationCountSnapshot; + var localActivations = GetLocalActivationCount(); + + var currentImbalance = 0; + currentImbalance = CalculateImbalance(Direction.Unspecified); + + var localHeap = new CandidateVertexMaxHeap(localSet); + var remoteHeap = new CandidateVertexMaxHeap(remoteSet); + + while (true) + { + if (localHeap.Count > 0 && remoteHeap.Count > 0) + { + var localVertex = localHeap.Peek(); + var remoteVertex = remoteHeap.Peek(); + + if (localVertex.AccumulatedTransferScore > remoteVertex.AccumulatedTransferScore) + { + if (TryMigrateLocalToRemote()) continue; + if (TryMigrateRemoteToLocal()) continue; + } + else if (localVertex.AccumulatedTransferScore < remoteVertex.AccumulatedTransferScore) + { + if (TryMigrateRemoteToLocal()) continue; + if (TryMigrateLocalToRemote()) continue; + } + else + { + // Other than testing scenarios with a handful of activations, it should be rare that this happens. If the transfer scores are equal, than we check the anticipated imbalances + // for both cases, and proceed with whichever lowers the overall imbalance, even though the other option could still be within the tolerance margin. + // The imbalance check is the first step micro-optimization, which doesn't necessarily mean that the migration direction (L2R, R2L) will happen, that is still + // determined within the migration methods. In case both anticipated imbalances are also equal, we have to pick one, and we stick for consistency with L2R in that case. + var l2r_anticipatedImbalance = CalculateImbalance(Direction.LocalToRemote); + var r2l_anticipatedImbalance = CalculateImbalance(Direction.RemoteToLocal); + + if (l2r_anticipatedImbalance <= r2l_anticipatedImbalance) + { + if (TryMigrateLocalToRemote()) continue; + if (TryMigrateRemoteToLocal()) continue; + } + else + { + if (TryMigrateRemoteToLocal()) continue; + if (TryMigrateLocalToRemote()) continue; + } + } + } + else + { + if (TryMigrateLocalToRemote()) continue; + if (TryMigrateRemoteToLocal()) continue; + + // Both heaps are empty, at this point we are done. + break; + } + } + + var unionSet = ImmutableArray.CreateBuilder(); + var mySet = ImmutableArray.CreateBuilder(); + var theirSet = ImmutableArray.CreateBuilder(); + foreach (var candidate in toMigrate) + { + if (candidate.TransferScore <= 0) + { + continue; + } + + if (candidate.Silo.Equals(Silo)) + { + // Add to the subset that should migrate to 'this' silo. + mySet.Add(candidate.Grain); + unionSet.Add(candidate.Grain); + } + else if (candidate.Silo.Equals(request.SendingSilo)) + { + // Add to the subset to send back to 'remote' silo (the actual migration will be handled there) + theirSet.Add(candidate.Grain); + unionSet.Add(candidate.Grain); + } + } + + await FinalizeProtocol(mySet.ToImmutable(), unionSet.ToImmutable(), isReceiver: true); + + return new(AcceptExchangeResponse.ResponseType.Success, theirSet.ToImmutable()); + + bool TryMigrateLocalToRemote() + { + if (localHeap.Count == 0) + { + return false; + } + + var anticipatedImbalance = CalculateImbalance(Direction.LocalToRemote); + if (anticipatedImbalance > currentImbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + { + return false; + } + + // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. + var chosenVertex = localHeap.Pop(); + if (chosenVertex.AccumulatedTransferScore <= 0) + { + return false; + } + + toMigrate.Add(new(chosenVertex.Id, request.SendingSilo, chosenVertex.AccumulatedTransferScore)); + + localActivations--; + remoteActivations++; + currentImbalance = anticipatedImbalance; + + var didUpdate = false; + foreach (var vertex in localHeap.UnorderedElements) + { + var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); + if (connectedVertex == default) + { + // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. + continue; + } + + // We add 'connectedVertex.TransferScore' to 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be remote to 'vertex' (because this is in the local heap). + vertex.AccumulatedTransferScore += connectedVertex.TransferScore; + didUpdate = true; + } + + if (didUpdate) + { + // Re-heapify the heap, since we changed priorities. + localHeap.Heapify(); + } + + didUpdate = false; + foreach (var vertex in remoteHeap.UnorderedElements) + { + var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); + if (connectedVertex == default) + { + // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. + continue; + } + + // We subtract 'connectedVertex.TransferScore' from 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be local to 'vertex' (because this is in the remote heap). + vertex.AccumulatedTransferScore -= connectedVertex.TransferScore; + didUpdate = true; + } + + if (didUpdate) + { + // Re-heapify the heap, since we changed priorities. + remoteHeap.Heapify(); + } + + return true; + } + + bool TryMigrateRemoteToLocal() + { + if (remoteHeap.Count == 0) + { + return false; + } + + var anticipatedImbalance = CalculateImbalance(Direction.RemoteToLocal); + if (anticipatedImbalance > currentImbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + { + return false; + } + + var chosenVertex = remoteHeap.Pop(); + if (chosenVertex.AccumulatedTransferScore <= 0) // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. + { + return false; + } + + toMigrate.Add(new(chosenVertex.Id, Silo, chosenVertex.AccumulatedTransferScore)); + + localActivations++; + remoteActivations--; + currentImbalance = anticipatedImbalance; + + var didUpdate = false; + foreach (var vertex in localHeap.UnorderedElements) + { + var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); + if (connectedVertex == default) + { + // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. + continue; + } + + // We subtract 'connectedVertex.TransferScore' from 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be local to 'vertex' (because this is in the local heap). + vertex.AccumulatedTransferScore -= connectedVertex.TransferScore; + didUpdate = true; + } + + if (didUpdate) + { + // Re-heapify the heap, since we changed priorities. + localHeap.Heapify(); + } + + didUpdate = false; + foreach (var vertex in remoteHeap.UnorderedElements) + { + var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); + if (connectedVertex == default) + { + // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. + continue; + } + + // We add 'connectedVertex.TransferScore' to 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be remote to 'vertex' (because this is in the remote heap) + vertex.AccumulatedTransferScore += connectedVertex.TransferScore; + didUpdate = true; + } + + if (didUpdate) + { + // Re-heapify the heap, since we changed priorities. + remoteHeap.Heapify(); + } + + return true; + } + + int CalculateImbalance(Direction direction) + { + (var rDelta, var lDelta) = direction switch + { + Direction.LocalToRemote => (1, -1), + Direction.RemoteToLocal => (-1, 1), + _ => (0, 0) + }; + + return Math.Abs(Math.Abs(remoteActivations + rDelta) - Math.Abs(localActivations + lDelta)); + } + } + finally + { + _logger.LogInformation("Computing transfer set took {Elapsed}.", sw.Elapsed); + _currentExchangeSilo = null; + } + } + + /// + /// + /// Initiates the actual migration process of to 'this' silo. + /// If it proceeds to update . + /// Updates the affected counters within to reflect all . + /// + /// + /// The grain ids to migrate. + /// All grains ids that were affected from both sides. + /// Is the caller, the protocol receiver or not. + private async Task FinalizeProtocol(ImmutableArray idsToMigrate, ImmutableArray affectedIds, bool isReceiver) + { + if (idsToMigrate.Length > 0) + { + // The protocol concluded that 'this' silo should take on 'set', so we hint to the director accordingly. + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo); + List migrationTasks = []; + + var sw1 = ValueStopwatch.StartNew(); + _logger.LogInformation("Telling {Count} grains to migrate", affectedIds.Length); + foreach (var grainId in idsToMigrate) + { + migrationTasks.Add(_grainFactory.GetGrain(grainId).Cast().MigrateOnIdle().AsTask()); + } + _logger.LogInformation("Telling {Count} grains to migrate took {Elapsed}", affectedIds.Length, sw1.Elapsed); + + try + { + await Task.WhenAll(migrationTasks); + } + catch (Exception) + { + // This should happen rarely, but at this point we cant really do much, as its out of our control. + // Even if some fail, at the end the algorithm will run again and eventually succeed with moving all activations were they belong. + var aggEx = new AggregateException(migrationTasks.Select(t => t.Exception).Where(ex => ex is not null)!); + LogErrorOnMigratingActivations(aggEx, Silo); + } + + if (isReceiver) + { + // Stamp this silos exchange for a potential next pair exchange request. + _lastExchangedStopwatch.Restart(); + } + } + + var sw = ValueStopwatch.StartNew(); + if (affectedIds.Length != 0) + { + // Avoid mutating the source while enumerating it. + var toRemove = new List(); + foreach (var (edge, count, error) in _edgeWeights.Elements) + { + if (edge.ContainsAny(affectedIds)) + { + toRemove.Add(edge); + } + } + + foreach (var edge in toRemove) + { + // Totally remove this counter, as one or both vertices has migrated. By not doing this it would skew results for the next protocol cycle. + // We remove only the affected counters, as there could be other counters that 'this' silo has connections with another silo (which is not part of this exchange cycle). + _edgeWeights.Remove(edge); + } + } + _logger.LogInformation("Removing transfer set from edge weights took {Elapsed}.", sw.Elapsed); + + LogProtocolFinalized(idsToMigrate.Length); + } + + private List<(SiloAddress Silo, ImmutableArray Candidates, long TransferScore)> CreateCandidateSets(ImmutableArray silos) + { + List<(SiloAddress Silo, ImmutableArray Candidates, long TransferScore)> candidateSets = new(silos.Length - 1); + var sw = ValueStopwatch.StartNew(); + var localVertices = CreateLocalVertexEdges().ToList(); + _logger.LogInformation("Computing local vertex edges took {Elapsed}.", sw.Elapsed); + + sw.Restart(); + foreach (var siloAddress in silos) + { + if (siloAddress.IsSameLogicalSilo(Silo)) + { + // We aren't going to exchange anything with ourselves, so skip this silo. + continue; + } + + var candidates = CreateCandidateSet(localVertices, siloAddress); + var totalAccTransferScore = candidates.Sum(x => x.AccumulatedTransferScore); + + candidateSets.Add(new(siloAddress, [.. candidates], totalAccTransferScore)); + } + + _logger.LogInformation("Computing candidate set per-silo took {Elapsed}.", sw.Elapsed); + + // Order them by the highest accumulated transfer score + candidateSets.Sort(static (a, b) => -a.TransferScore.CompareTo(b.TransferScore)); + + return candidateSets; + } + + private List CreateCandidateSet(IEnumerable edges, SiloAddress otherSilo) + { + Debug.Assert(otherSilo.IsSameLogicalSilo(Silo) is false); + + List candidates = []; + + // We skip types that cant be migrated. Instead the same edge will be recorded from the receiver, so its hosting silo will add it as a candidate to be migrated (over here). + // We are sure that the receiver is an migratable grain, because the gateway forbids edges that have non-migratable vertices on both sides. + foreach (var grouping in edges + .Where(x => x.IsMigratable) + .GroupBy(x => x.SourceId)) + { + var accLocalScore = 0L; + var accRemoteScore = 0L; + + foreach (var entry in grouping) + { + if (entry.Direction == Direction.LocalToLocal) + { + // Since its L2L, it means the partner silo will be 'this' silo, so we don't need to filter by the partner silo. + accLocalScore += entry.Weight; + } + else if (entry.TargetSilo.Equals(otherSilo) && entry.Direction is Direction.RemoteToLocal or Direction.LocalToRemote) + { + // We need to filter here by 'otherSilo' since any L2R or R2L edge can be between the current vertex and a vertex in a silo that is not in 'otherSilo'. + accRemoteScore += entry.Weight; + } + } + + var totalAccScore = accRemoteScore - accLocalScore; + if (totalAccScore <= 0) + { + // We skip vertices for which local calls outweigh the remote once. + continue; + } + + var connVertices = ImmutableArray.CreateBuilder(); + foreach (var x in grouping) + { + // Note that the connected vertices can be of types which are not migratable, it is important to keep them, + // as they too impact the migration cost of the current candidate vertex, especially if they are local to the candidate + // as those calls would be potentially converted to remote calls, after the migration of the current candidate. + // 'Weight' here represent the weight of a single edge, not the accumulated like above. + connVertices.Add(new CandidateConnectedVertex(x.TargetId, x.Weight)); + } + + CandidateVertex candidate = new() + { + Id = grouping.Key, + AccumulatedTransferScore = totalAccScore, + ConnectedVertices = connVertices.ToImmutable() + }; + + candidates.Add(candidate); + } + + return candidates; + } + + /// + /// Creates a collection of 'local' vertex edges. Multiple entries can have the same Id. + /// + /// The is guaranteed to belong to a grain that is local to this silo, while might belong to a local or remote silo. + private IEnumerable CreateLocalVertexEdges() + { + foreach (var (edge, count, error) in _edgeWeights.Elements) + { + if (count == 0) + { + continue; + } + + var vertexEdge = CreateVertexEdge(new WeightedEdge(edge, count)); + yield return vertexEdge; + + if (vertexEdge.Direction == Direction.LocalToLocal) + { + // The reason we do this flipping is because when the edge is Local-to-Local, we have 2 grains that are linked via an communication edge. + // Once an edge exists it means 2 grains are temporally linked, this means that there is a cost associated to potentially move either one of them. + // Since the construction of the candidate set takes into account also local connection (which increases the cost of migration), we need + // to take into account the edge not only from a source's perspective, but also the target's one, as it too will take part on the candidate set. + var flippedEdge = CreateVertexEdge(new WeightedEdge(edge.Flip(), count)); + yield return flippedEdge; + } + } + + VertexEdge CreateVertexEdge(in WeightedEdge counter) + { + var direction = Direction.Unspecified; + + direction = IsSourceThisSilo(counter) + ? IsTargetThisSilo(counter) ? Direction.LocalToLocal : Direction.LocalToRemote + : Direction.RemoteToLocal; + + Debug.Assert(direction != Direction.Unspecified); // this can only occur when both: source and target are remote (which can not happen) + + return direction switch + { + Direction.LocalToLocal => new( + SourceId: counter.Edge.Source.Id, // 'local' vertex was the 'source' of the communication + TargetId: counter.Edge.Target.Id, + IsMigratable: counter.Edge.Source.IsMigratable, + TargetSilo: Silo, // the partner was 'local' (note: this.Silo = Source.Silo = Target.Silo) + Direction: direction, + Weight: counter.Weight), + + Direction.LocalToRemote => new( + SourceId: counter.Edge.Source.Id, // 'local' vertex was the 'source' of the communication + TargetId: counter.Edge.Target.Id, + IsMigratable: counter.Edge.Source.IsMigratable, + TargetSilo: counter.Edge.Target.Silo, // the partner was 'remote' + Direction: direction, + Weight: counter.Weight), + + Direction.RemoteToLocal => new( + SourceId: counter.Edge.Target.Id, // 'local' vertex was the 'target' of the communication + TargetId: counter.Edge.Source.Id, + IsMigratable: counter.Edge.Target.IsMigratable, + TargetSilo: counter.Edge.Source.Silo, // the partner was 'remote' + Direction: direction, + Weight: counter.Weight), + + _ => throw new UnreachableException($"The edge direction {direction} is out of range.") + }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + bool IsSourceThisSilo(in WeightedEdge counter) => counter.Edge.Source.Silo.IsSameLogicalSilo(Silo); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + bool IsTargetThisSilo(in WeightedEdge counter) => counter.Edge.Target.Silo.IsSameLogicalSilo(Silo); + } + + public void Participate(ISiloLifecycle observer) + { + // Start when the silo becomes active. + observer.Subscribe( + nameof(ActivationRebalancer), + ServiceLifecycleStage.Active, + OnActiveStart, + ct => Task.CompletedTask); + + // Stop when the silo stops application services. + observer.Subscribe( + nameof(ActivationRebalancer), + ServiceLifecycleStage.ApplicationServices, + ct => Task.CompletedTask, + StopProcessingEdgesAsync); + } + + void IDisposable.Dispose() + { + base.Dispose(); + _enableMessageSampling = false; + _siloStatusOracle.UnSubscribeFromSiloStatusEvents(this); + _shutdownCts.Cancel(); + } + + void ISiloStatusListener.SiloStatusChangeNotification(SiloAddress updatedSilo, SiloStatus status) + { + _enableMessageSampling = _siloStatusOracle.GetActiveSilos().Length > 1; + } + + private enum Direction : byte + { + Unspecified = 0, + LocalToLocal = 1, + LocalToRemote = 2, + RemoteToLocal = 3 + } + + /// + /// Represents a connection between 2 vertices. + /// + /// The id of the grain it represents. + /// The id of the connected vertex (the one the communication took place with). + /// Specifies if the vertex with is a migratable type. + /// The silo partner which interacted with the silo of vertex with . + /// The edge's direction + /// The number of estimated messages exchanged between and . + private readonly record struct VertexEdge(GrainId SourceId, GrainId TargetId, bool IsMigratable, SiloAddress TargetSilo, Direction Direction, long Weight); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/DefaultImbalanceRule.cs b/src/Orleans.Runtime/Placement/Rebalancing/DefaultImbalanceRule.cs new file mode 100644 index 0000000000..d00208bf6a --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/DefaultImbalanceRule.cs @@ -0,0 +1,56 @@ +using System; +using System.Linq; +using System.Collections.Concurrent; +using Orleans.Placement.Rebalancing; +using System.Threading.Tasks; +using System.Threading; + +namespace Orleans.Runtime.Placement.Rebalancing; + +/// +/// Tolerance rule which is aware of the cluster size. +/// +internal sealed class DefaultImbalanceRule(ISiloStatusOracle siloStatusOracle) : IImbalanceToleranceRule, + ILifecycleParticipant, ILifecycleObserver, ISiloStatusListener +{ + private const double Baseline = 10.1d; + private readonly object _lock = new(); + private readonly ConcurrentDictionary _silos = new(); + private readonly ISiloStatusOracle _siloStatusOracle = siloStatusOracle; + + private uint _allowedImbalance = 0; + + public bool IsSatisfiedBy(uint imbalance) => imbalance <= _allowedImbalance; + + public void SiloStatusChangeNotification(SiloAddress silo, SiloStatus status) + { + _ = _silos.AddOrUpdate(silo, static (_, arg) => arg, static (_, _, arg) => arg, status); + lock (_lock) + { + var activeSilos = _silos.Count(s => s.Value == SiloStatus.Active); + var percentageOfBaseline = 100d / (1 + Math.Exp(0.07d * activeSilos - 4.8d)); // inverted sigmoid + if (percentageOfBaseline < 10d) + { + percentageOfBaseline = 10d; + } + + // silos: 2 => tolerance: ~ 1000 + // silos: 100 => tolerance: ~ 100 + _allowedImbalance = (uint)Math.Round(Baseline * percentageOfBaseline, 0); + } + } + + public void Participate(ISiloLifecycle lifecycle) + => lifecycle.Subscribe(nameof(DefaultImbalanceRule), ServiceLifecycleStage.ApplicationServices, this); + + public Task OnStart(CancellationToken cancellationToken = default) + { + _siloStatusOracle.SubscribeToSiloStatusEvents(this); + return Task.CompletedTask; + } + public Task OnStop(CancellationToken cancellationToken = default) + { + _siloStatusOracle.UnSubscribeFromSiloStatusEvents(this); + return Task.CompletedTask; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs b/src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs new file mode 100644 index 0000000000..00b0846754 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs @@ -0,0 +1,397 @@ +#nullable enable +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using Orleans.Placement.Rebalancing; + +namespace Orleans.Runtime.Placement.Rebalancing; + +internal sealed class FrequentEdgeCounter(int capacity) : FrequentItemCollection(capacity) +{ + protected override ulong GetKey(in Edge element) => (ulong)element.Source.Id.GetUniformHashCode() << 32 | element.Target.Id.GetUniformHashCode(); + public void Clear() => ClearCore(); + public void Remove(in Edge element) => RemoveCore(GetKey(element)); +} + +// This is Implementation of "Filtered Space-Saving" from "Finding top-k elements in data streams" +// by Nuno Homem & Joao Paulo Carvalho (https://www.hlt.inesc-id.pt/~fmmb/references/misnis.ref0a.pdf). +// In turn, this is a modification of the "Space-Saving" algorithm by Metwally, Agrawal, and Abbadi, +// Described in "Efficient Computation of Frequent and Top-k Elements in Data Streams" (https://www.cs.emory.edu/~cheung/Courses/584/Syllabus/papers/Frequency-count/2005-Metwally-Top-k-elements.pdf). +// This is implemented using an in-lined version of .NET's PriorityQueue which has been modified +// to support incrementing a value and with an index mapping key hashes to heap indexes. +internal abstract class FrequentItemCollection(int capacity) where TElement : notnull where TKey : notnull +{ + /// + /// Represents an implicit heap-ordered complete d-ary tree, stored as an array. + /// + private Counter[] _heap = []; + + /// + /// A dictionary that maps the hash of a key to its index in the heap. + /// + private readonly Dictionary _heapIndex = []; + + /// + /// The number of nodes in the heap. + /// + private int _heapSize; + + /// + /// Specifies the arity of the d-ary heap, which here is quaternary. + /// It is assumed that this value is a power of 2. + /// + private const int Arity = 4; + + /// + /// The binary logarithm of . + /// + private const int Log2Arity = 2; + + /// + /// Contains count estimates for keys that are not being tracked, indexed by the hash of the key. + /// Collisions are expected. + /// + private readonly uint[] _sketch = new uint[GetSketchSize(capacity)]; + + /// + /// Gets the number of elements contained in the . + /// + public int Count => _heapSize; + + /// + /// Gets the number of elements which the will track. + /// + public int Capacity { get; } = capacity; + +#if DEBUG + static FrequentItemCollection() + { + Debug.Assert(Log2Arity > 0 && Math.Pow(2, Log2Arity) == Arity); + } +#endif + + /// + /// Returns a collection of up to keys, along with their count estimates, in unspecified order. + /// + public ElementEnumerator Elements => new(this); + + protected abstract TKey GetKey(in TElement element); + + public void Add(in TElement element) + { + const int Increment = 1; + var nodeIndexHash = GetKey(element); + + // Increase count of a key that is already being tracked. + // There is a minute chance of a hash collision, which is deemed acceptable and ignored. + if (_heapIndex.TryGetValue(nodeIndexHash, out var index)) + { + ref var counter = ref _heap[index]; + counter.Count += Increment; + MoveUpHeap(counter, index, nodeIndexHash); + return; + } + + // Key is not being tracked, but can fit in the top K, so add it. + if (Count < Capacity) + { + InsertHeap(new Counter(element, Increment, error: 0), nodeIndexHash); + return; + } + + var min = _heap[0]; + + // Filter out values which are estimated to have appeared less frequently than the minimum. + var sketchMask = _sketch.Length - 1; + var sketchHash = nodeIndexHash.GetHashCode(); + var countEstimate = _sketch[sketchHash & sketchMask]; + if (countEstimate + Increment < min.Count) + { + // Increase the count estimate. + _sketch[sketchHash & sketchMask] += Increment; + return; + } + + // Remove the minimum element from the hash index. + var minIndexHash = GetKey(min.Element); + _heapIndex.Remove(minIndexHash); + + // While evicting the minimum element, update its counter in the sketch to improve the chance of it + // passing the filter in the future. + var minHash = minIndexHash.GetHashCode(); + _sketch[minHash & sketchMask] = min.Count; + + // Push the new element in place of the last and move it down until it's in position. + MoveDownHeap(new Counter(element, countEstimate + Increment, error: countEstimate), 0, nodeIndexHash); + } + + /// + /// Removes the counter corresponding to the specified hash. + /// + /// The key of the value to remove. + /// if matching entry was found and removed, otherwise. + protected bool RemoveCore(TKey key) + { + // Remove the element from the heap index + if (!_heapIndex.Remove(key, out var index)) + { + return false; + } + + // Remove the element from the heap + var nodes = _heap; + var newSize = --_heapSize; + if (index < newSize) + { + // We're removing an element from the middle of the heap. + // Pop the last element in the collection and sift downward from the removed index. + var lastNode = nodes[newSize]; + + MoveDownHeap(lastNode, index, GetKey(lastNode.Element)); + } + + nodes[newSize] = default; + + // Remove the element from the sketch + var sketchMask = _sketch.Length - 1; + var sketchHash = key.GetHashCode(); + _sketch[sketchHash & sketchMask] = 0; + + return true; + } + + protected void ClearCore() + { + Array.Clear(_heap, 0, _heapSize); + _heapIndex.Clear(); + Array.Clear(_sketch); + _heapSize = 0; + } + + private static int GetSketchSize(int capacity) + { + // Suggested constants in the paper "Finding top-k elements in data streams", chap 6. equation (24) + // Round to nearest power of 2 for cheaper binning without modulo + const int SketchEntriesPerHeapEntry = 6; + + return 1 << (32 - int.LeadingZeroCount(capacity * SketchEntriesPerHeapEntry)); + } + + /// + /// Adds the specified element to the . + /// + /// The element to add. + private void InsertHeap(Counter element, TKey key) + { + // Virtually add the node at the end of the underlying array. + // Note that the node being enqueued does not need to be physically placed + // there at this point, as such an assignment would be redundant. + + var currentSize = _heapSize; + + if (_heap.Length == currentSize) + { + GrowHeap(currentSize + 1); + } + + _heapSize = currentSize + 1; + + MoveUpHeap(element, currentSize, key); + } + + /// + /// Grows the priority queue to match the specified min capacity. + /// + private void GrowHeap(int minCapacity) + { + Debug.Assert(_heap.Length < minCapacity); + + const int GrowFactor = 2; + const int MinimumGrow = 4; + + var newCapacity = GrowFactor * _heap.Length; + + // Allow the queue to grow to maximum possible capacity (~2G elements) before encountering overflow. + // Note that this check works even when _nodes.Length overflowed thanks to the (uint) cast + if ((uint)newCapacity > Array.MaxLength) newCapacity = Array.MaxLength; + + // Ensure minimum growth is respected. + newCapacity = Math.Max(newCapacity, _heap.Length + MinimumGrow); + + // If the computed capacity is still less than specified, set to the original argument. + // Capacities exceeding Array.MaxLength will be surfaced as OutOfMemoryException by Array.Resize. + if (newCapacity < minCapacity) newCapacity = minCapacity; + + Array.Resize(ref _heap, newCapacity); + } + + /// + /// Gets the index of an element's parent. + /// + private static int GetParentIndex(int index) => (index - 1) >> Log2Arity; + + /// + /// Gets the index of the first child of an element. + /// + private static int GetFirstChildIndex(int index) => (index << Log2Arity) + 1; + + /// + /// Moves a node up in the tree to restore heap order. + /// + private void MoveUpHeap(Counter node, int nodeIndex, TKey nodeKey) + { + // Instead of swapping items all the way to the root, we will perform + // a similar optimization as in the insertion sort. + + Debug.Assert(0 <= nodeIndex && nodeIndex < _heapSize); + + var nodes = _heap; + var hashIndex = _heapIndex; + + while (nodeIndex > 0) + { + var parentIndex = GetParentIndex(nodeIndex); + var parent = nodes[parentIndex]; + + if (node.CompareTo(parent) < 0) + { + nodes[nodeIndex] = parent; + hashIndex[GetKey(parent.Element)] = nodeIndex; + nodeIndex = parentIndex; + } + else + { + break; + } + } + + nodes[nodeIndex] = node; + hashIndex[nodeKey] = nodeIndex; + } + + /// + /// Moves a node down in the tree to restore heap order. + /// + private void MoveDownHeap(Counter node, int nodeIndex, TKey nodeKey) + { + // The node to move down will not actually be swapped every time. + // Rather, values on the affected path will be moved up, thus leaving a free spot + // for this value to drop in. Similar optimization as in the insertion sort. + + Debug.Assert(0 <= nodeIndex && nodeIndex < _heapSize); + + var nodes = _heap; + var size = _heapSize; + var hashIndex = _heapIndex; + + int i; + while ((i = GetFirstChildIndex(nodeIndex)) < size) + { + // Find the child node with the minimal priority + var minChild = nodes[i]; + var minChildIndex = i; + + var childIndexUpperBound = Math.Min(i + Arity, size); + while (++i < childIndexUpperBound) + { + var nextChild = nodes[i]; + if (nextChild.CompareTo(minChild) < 0) + { + minChild = nextChild; + minChildIndex = i; + } + } + + // Heap property is satisfied; insert node in this location. + if (node.CompareTo(minChild) <= 0) + { + break; + } + + // Move the minimal child up by one node and continue recursively from its location. + nodes[nodeIndex] = minChild; + hashIndex[GetKey(minChild.Element)] = nodeIndex; + nodeIndex = minChildIndex; + } + + hashIndex[nodeKey] = nodeIndex; + nodes[nodeIndex] = node; + } + + private struct Counter(TElement element, uint count, uint error) : IComparable + { + public readonly TElement Element = element; + public uint Count = count; + public uint Error = error; + + public readonly int CompareTo(Counter other) => ((ulong)Count << 32 | uint.MaxValue - Error).CompareTo((ulong)other.Count << 32 | uint.MaxValue - other.Error); + + public override readonly string ToString() => $"{Element}: Count: {Count} Error: {Error}"; + } + + /// + /// Enumerates the element and priority pairs of a , + /// without any ordering guarantees. + /// + public struct ElementEnumerator : IEnumerator<(TElement Element, uint Count, uint Error)>, IEnumerable<(TElement Element, uint Count, uint Error)> + { + private readonly FrequentItemCollection _heap; + private int _index; + private Counter _current; + + internal ElementEnumerator(FrequentItemCollection heap) + { + _heap = heap; + _index = 0; + _current = default; + } + + /// + /// Releases all resources used by the . + /// + public readonly void Dispose() { } + + /// + /// Advances the enumerator to the next element of the heap. + /// + /// if the enumerator was successfully advanced to the next element; if the enumerator has passed the end of the collection. + public bool MoveNext() + { + var localHeap = _heap; + + if ((uint)_index < (uint)localHeap._heapSize) + { + _current = localHeap._heap[_index]; + _index++; + return true; + } + + return MoveNextRare(); + } + + private bool MoveNextRare() + { + _index = _heap._heapSize + 1; + _current = default; + return false; + } + + /// + /// Gets the element at the current position of the enumerator. + /// + public readonly (TElement Element, uint Count, uint Error) Current => (_current.Element, _current.Count, _current.Error); + + readonly object IEnumerator.Current => _current; + + void IEnumerator.Reset() + { + _index = 0; + _current = default; + } + + public readonly ElementEnumerator GetEnumerator() => this; + readonly IEnumerator<(TElement Element, uint Count, uint Error)> IEnumerable<(TElement Element, uint Count, uint Error)>.GetEnumerator() => this; + readonly IEnumerator IEnumerable.GetEnumerator() => this; + } +} diff --git a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs new file mode 100644 index 0000000000..a0ee133e7d --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs @@ -0,0 +1,281 @@ +#nullable enable +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Orleans.Placement.Rebalancing; + +namespace Orleans.Runtime.Placement.Rebalancing; + +internal sealed class CandidateVertexMaxHeap(ICollection values) : MaxHeap(values) +{ + protected override int Compare(CandidateVertex left, CandidateVertex right) => -left.AccumulatedTransferScore.CompareTo(right.AccumulatedTransferScore); +} + +/// +/// Represents a max heap. +/// +/// Specifies the type of elements in the heap. +/// +/// Implements an array-backed quaternary max-heap. +/// Elements with the lowest priority get removed first. +/// +[DebuggerDisplay("Count = {Count}")] +internal abstract class MaxHeap where TElement : notnull +{ + /// + /// Represents an implicit heap-ordered complete d-ary tree, stored as an array. + /// + private readonly TElement?[] _nodes; + + /// + /// The number of nodes in the heap. + /// + private int _size; + + /// + /// Specifies the arity of the d-ary heap, which here is quaternary. + /// It is assumed that this value is a power of 2. + /// + private const int Arity = 4; + + /// + /// The binary logarithm of . + /// + private const int Log2Arity = 2; + + protected abstract int Compare(TElement left, TElement right); + +#if DEBUG + static MaxHeap() + { + Debug.Assert(Log2Arity > 0 && Math.Pow(2, Log2Arity) == Arity); + } +#endif + + /// + /// Initializes a new instance of the class + /// that is populated with the specified elements and priorities. + /// + /// The pairs of elements and priorities with which to populate the queue. + /// + /// The specified argument was . + /// + /// + /// Constructs the heap using a heapify operation, + /// which is generally faster than enqueuing individual elements sequentially. + /// + public MaxHeap(ICollection items) + { + ArgumentNullException.ThrowIfNull(items); + + _size = items.Count; + var nodes = new TElement[_size]; + items.CopyTo(nodes, 0); + _nodes = nodes; + + if (_size > 1) + { + Heapify(); + } + } + + /// + /// Gets the number of elements contained in the . + /// + public int Count => _size; + + /// + /// Returns the maximal element from the without removing it. + /// + /// The is empty. + /// The maximal element of the . + public TElement Peek() + { + if (_size == 0) + { + throw new InvalidOperationException("Collection is empty."); + } + + return _nodes[0]!; + } + + /// + /// Removes and returns the maximal element from the . + /// + /// The queue is empty. + /// The maximal element of the . + public TElement Pop() + { + if (_size == 0) + { + throw new InvalidOperationException("Collection is empty."); + } + + var element = _nodes[0]!; + RemoveRootNode(); + return element; + + void RemoveRootNode() + { + var lastNodeIndex = --_size; + + if (lastNodeIndex > 0) + { + var lastNode = _nodes[lastNodeIndex]!; + MoveDown(lastNode, 0); + } + + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + _nodes[lastNodeIndex] = default!; + } + } + } + + /// + /// Gets the index of an element's parent. + /// + private static int GetParentIndex(int index) => (index - 1) >> Log2Arity; + + /// + /// Gets the index of the first child of an element. + /// + private static int GetFirstChildIndex(int index) => (index << Log2Arity) + 1; + + /// + /// Converts an unordered list into a heap. + /// + public void Heapify() + { + // Leaves of the tree are in fact 1-element heaps, for which there + // is no need to correct them. The heap property needs to be restored + // only for higher nodes, starting from the first node that has children. + // It is the parent of the very last element in the array. + + var nodes = _nodes; + var lastParentWithChildren = GetParentIndex(_size - 1); + for (var index = lastParentWithChildren; index >= 0; --index) + { + MoveDown(nodes[index]!, index); + } + } + + /// + /// Gets the elements in this collection with specified order. + /// + public UnorderedElementEnumerable UnorderedElements => new(this); + + /// + /// Moves a node down in the tree to restore heap order. + /// + private void MoveDown(TElement node, int nodeIndex) + { + // The node to move down will not actually be swapped every time. + // Rather, values on the affected path will be moved up, thus leaving a free spot + // for this value to drop in. Similar optimization as in the insertion sort. + + Debug.Assert(0 <= nodeIndex && nodeIndex < _size); + + var nodes = _nodes; + var size = _size; + + int i; + while ((i = GetFirstChildIndex(nodeIndex)) < size) + { + // Find the child node with the maximal priority + var minChild = nodes[i]!; + var minChildIndex = i; + + var childIndexUpperBound = Math.Min(i + Arity, size); + while (++i < childIndexUpperBound) + { + var nextChild = nodes[i]!; + if (Compare(nextChild, minChild) < 0) + { + minChild = nextChild; + minChildIndex = i; + } + } + + // Heap property is satisfied; insert node in this location. + if (Compare(node, minChild) <= 0) + { + break; + } + + // Move the maximal child up by one node and + // continue recursively from its location. + nodes[nodeIndex] = minChild; + nodeIndex = minChildIndex; + } + + nodes[nodeIndex] = node; + } + + /// + /// Enumerates the element and priority pairs of a + /// without any ordering guarantees. + /// + public struct UnorderedElementEnumerable : IEnumerator, IEnumerable + { + private readonly MaxHeap _heap; + private int _index; + private TElement? _current; + + internal UnorderedElementEnumerable(MaxHeap heap) + { + _heap = heap; + _index = 0; + _current = default; + } + + /// + /// Releases all resources used by the . + /// + public readonly void Dispose() { } + + /// + /// Advances the enumerator to the next element of the heap. + /// + /// if the enumerator was successfully advanced to the next element; if the enumerator has passed the end of the collection. + public bool MoveNext() + { + var localHeap = _heap; + + if ((uint)_index < (uint)localHeap._size) + { + _current = localHeap._nodes[_index]; + _index++; + return true; + } + + return MoveNextRare(); + } + + private bool MoveNextRare() + { + _index = _heap._size + 1; + _current = default; + return false; + } + + /// + /// Gets the element at the current position of the enumerator. + /// + public readonly TElement Current => _current ?? throw new InvalidOperationException("Current element is not valid."); + + readonly object IEnumerator.Current => Current; + + public readonly UnorderedElementEnumerable GetEnumerator() => this; + readonly IEnumerator IEnumerable.GetEnumerator() => this; + void IEnumerator.Reset() + { + _index = 0; + _current = default; + } + + readonly IEnumerator IEnumerable.GetEnumerator() => this; + } +} diff --git a/src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs new file mode 100644 index 0000000000..82127cb3c0 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs @@ -0,0 +1,129 @@ +#nullable enable +using Orleans.Metadata; +using System; +using System.Collections.Concurrent; +using System.Collections.Frozen; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Orleans.Runtime.Placement.Rebalancing; + +internal interface IRebalancingMessageFilter +{ + bool IsAcceptable(Message message, out bool isSenderMigratable, out bool isTargetMigratable); +} + +internal sealed class RebalancingMessageFilter( + PlacementStrategyResolver strategyResolver, + IClusterManifestProvider clusterManifestProvider, + TimeProvider timeProvider) : IRebalancingMessageFilter +{ + private readonly GrainManifest _localManifest = clusterManifestProvider.LocalGrainManifest; + private readonly PlacementStrategyResolver _strategyResolver = strategyResolver; + private readonly TimeProvider _timeProvider = timeProvider; + private readonly ConcurrentDictionary _migratableStatuses = new(); + private FrozenDictionary? _migratableStatusesCache; + private long _lastRegeneratedCacheTimestamp = timeProvider.GetTimestamp(); + + public bool IsAcceptable(Message message, out bool isSenderMigratable, out bool isTargetMigratable) + { + isSenderMigratable = false; + isTargetMigratable = false; + + // Ignore system messages + if (message.IsSystemMessage) + { + return false; + } + + // It must have a direction, and must not be a 'response' as it would skew analysis. + if (message.HasDirection is false || message.Direction == Message.Directions.Response) + { + return false; + } + + // Sender and target need to be fully addressable to know where to move to or towards. + if (!message.IsSenderFullyAddressed || !message.IsTargetFullyAddressed) + { + return false; + } + + // There are some edge cases when this can happen i.e. a grain invoking another one of its methods via AsReference<>, but we still exclude it + // as wherever this grain would be located in the cluster, it would always be a local call (since it targets itself), this would add negative transfer cost + // which would skew a potential relocation of this grain, while it shouldn't, because whenever this grain is located, it would still make local calls to itself. + if (message.SendingGrain == message.TargetGrain) + { + return false; + } + + // Ignore rebalancer messages: either to another rebalancer, or when executing migration requests to activations. + if (IsRebalancer(message.SendingGrain.Type) || IsRebalancer(message.TargetGrain.Type)) + { + return false; + } + + isSenderMigratable = IsMigratable(message.SendingGrain.Type); + isTargetMigratable = IsMigratable(message.TargetGrain.Type); + + // If both are not migratable types we ignore this. But if one of them is not, then we allow passing, as we wish to move grains closer to them, as with any type of grain. + if (!isSenderMigratable && !isTargetMigratable) + { + return false; + } + + return true; + + bool IsRebalancer(GrainType grainType) => grainType.Equals(Constants.ActivationRebalancerType); + + bool IsMigratable(GrainType grainType) + { + var hash = grainType.GetUniformHashCode(); + if (_migratableStatusesCache is { } cache && cache.TryGetValue(hash, out var isMigratable)) + { + return isMigratable; + } + + return IsMigratableRare(grainType, hash); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + bool IsStatelessWorker(GrainType grainType) => + _strategyResolver.GetPlacementStrategy(grainType).GetType() == typeof(StatelessWorkerPlacement); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + bool IsImmovable(GrainType grainType) + { + if (_localManifest.Grains.TryGetValue(grainType, out var props)) + { + // If there is no 'Immovable' property, it is not immovable. + // If the value fails to parse, assume it's immovable. + // If the value is true, it's immovable. + return props.Properties.TryGetValue(WellKnownGrainTypeProperties.Immovable, out var value) && (!bool.TryParse(value, out var result) || result); + } + + // Assume unknown grains are immovable. + return true; + } + + bool IsMigratableRare(GrainType grainType, uint hash) + { + // _migratableStatuses holds statuses for each grain type if its migratable type or not, so we can make fast lookups. + // since we don't anticipate a huge number of grain *types*, i think its just fine to have this in place as fast-check. + if (!_migratableStatuses.TryGetValue(hash, out var isMigratable)) + { + isMigratable = !(grainType.IsClient() || grainType.IsSystemTarget() || grainType.IsGrainService() || IsStatelessWorker(grainType) || IsImmovable(grainType)); + _migratableStatuses.TryAdd(hash, isMigratable); + } + + // Regenerate the cache periodically. + var currentTimestamp = _timeProvider.GetTimestamp(); + if (_timeProvider.GetElapsedTime(_lastRegeneratedCacheTimestamp, currentTimestamp) > TimeSpan.FromSeconds(5)) + { + _migratableStatusesCache = _migratableStatuses.ToFrozenDictionary(); + _lastRegeneratedCacheTimestamp = currentTimestamp; + } + + return isMigratable; + } + } + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs b/src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs new file mode 100644 index 0000000000..47566f94fd --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs @@ -0,0 +1,22 @@ +using System.Diagnostics; +using Orleans.Placement.Rebalancing; + +namespace Orleans.Runtime.Placement.Rebalancing; + +#nullable enable + +/// +/// Represents a weighted . +/// +[DebuggerDisplay("Value {Value} | Edge = {Edge}")] +internal readonly struct WeightedEdge(Edge edge, long weight) +{ + public readonly Edge Edge = edge; + + public readonly long Weight = weight; + + /// + /// Returns a copy of this but with flipped sources and targets. + /// + public WeightedEdge Flip() => new(Edge.Flip(), Weight); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Properties/AssemblyInfo.cs b/src/Orleans.Runtime/Properties/AssemblyInfo.cs index dc9bc9a6bd..4c2864c6a2 100644 --- a/src/Orleans.Runtime/Properties/AssemblyInfo.cs +++ b/src/Orleans.Runtime/Properties/AssemblyInfo.cs @@ -11,6 +11,7 @@ [assembly: InternalsVisibleTo("Tester.AdoNet")] [assembly: InternalsVisibleTo("TesterInternal")] [assembly: InternalsVisibleTo("TestInternalGrains")] +[assembly: InternalsVisibleTo("Benchmarks")] // Mocking libraries [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] \ No newline at end of file diff --git a/src/Orleans.Runtime/Utilities/StripedMpscBuffer.cs b/src/Orleans.Runtime/Utilities/StripedMpscBuffer.cs new file mode 100644 index 0000000000..6df8776221 --- /dev/null +++ b/src/Orleans.Runtime/Utilities/StripedMpscBuffer.cs @@ -0,0 +1,425 @@ +using System; +using System.Linq; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Threading; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace Orleans.Runtime.Utilities; + +/// +/// Provides a striped bounded buffer. Add operations use thread ID to index into +/// the underlying array of buffers, and if TryAdd is contended the thread ID is +/// rehashed to select a different buffer to retry up to 3 times. Using this approach +/// writes scale linearly with number of concurrent threads. +/// +[DebuggerDisplay("Count = {Count}/{Capacity}")] +internal sealed class StripedMpscBuffer where T : class +{ + private const int MaxAttempts = 3; + + private readonly MpscBoundedBuffer[] _buffers; + + /// + /// Initializes a new instance of the StripedMpscBuffer class with the specified stripe count and buffer size. + /// + /// The stripe count. + /// The buffer size. + public StripedMpscBuffer(int stripeCount, int bufferSize) + { + _buffers = new MpscBoundedBuffer[stripeCount]; + + for (var i = 0; i < stripeCount; i++) + { + _buffers[i] = new MpscBoundedBuffer(bufferSize); + } + } + + /// + /// Gets the number of items contained in the buffer. + /// + public int Count => _buffers.Sum(b => b.Count); + + /// + /// The bounded capacity. + /// + public int Capacity => _buffers.Length * _buffers[0].Capacity; + + /// + /// Drains the buffer into the specified array. + /// + /// The output buffer + /// The number of items written to the output buffer. + /// + /// Thread safe for single try take/drain + multiple try add. + /// + public int DrainTo(T[] outputBuffer) => DrainTo(outputBuffer.AsSpan()); + + /// + /// Drains the buffer into the specified span. + /// + /// The output buffer + /// The number of items written to the output buffer. + /// + /// Thread safe for single try take/drain + multiple try add. + /// + public int DrainTo(Span outputBuffer) + { + var count = 0; + + for (var i = 0; i < _buffers.Length; i++) + { + if (count == outputBuffer.Length) + { + break; + } + + var segment = outputBuffer[count..]; + + count += _buffers[i].DrainTo(segment); + } + + return count; + } + + /// + /// Tries to add the specified item. + /// + /// The item to be added. + /// A BufferStatus value indicating whether the operation succeeded. + /// + /// Thread safe. + /// + public BufferStatus TryAdd(T item) + { + var z = BitOps.Mix64((ulong)Environment.CurrentManagedThreadId); + var inc = (int)(z >> 32) | 1; + var h = (int)z; + + var mask = _buffers.Length - 1; + + var result = BufferStatus.Empty; + + for (var i = 0; i < MaxAttempts; i++) + { + result = _buffers[h & mask].TryAdd(item); + + if (result == BufferStatus.Success) + { + break; + } + + h += inc; + } + + return result; + } + + /// + /// Removes all values from the buffer. + /// + /// + /// Not thread safe. + /// + public void Clear() + { + for (var i = 0; i < _buffers.Length; i++) + { + _buffers[i].Clear(); + } + } +} + +/// +/// Provides a multi-producer, single-consumer thread-safe ring buffer. When the buffer is full, +/// TryAdd fails and returns false. When the buffer is empty, TryTake fails and returns false. +/// +/// Based on the BoundedBuffer class in the Caffeine library by ben.manes@gmail.com (Ben Manes). +[DebuggerDisplay("Count = {Count}/{Capacity}")] +internal sealed class MpscBoundedBuffer where T : class +{ + private T[] _buffer; + private readonly int _mask; + private PaddedHeadAndTail _headAndTail; // mutable struct, don't mark readonly + + /// + /// Initializes a new instance of the MpscBoundedBuffer class with the specified bounded capacity. + /// + /// The bounded length. + /// + public MpscBoundedBuffer(int boundedLength) + { + ArgumentOutOfRangeException.ThrowIfLessThan(boundedLength, 0); + + // must be power of 2 to use & slotsMask instead of % + boundedLength = BitOps.CeilingPowerOfTwo(boundedLength); + + _buffer = new T[boundedLength]; + _mask = boundedLength - 1; + } + + /// + /// The bounded capacity. + /// + public int Capacity => _buffer.Length; + + /// + /// Gets the number of items contained in the buffer. + /// + public int Count + { + get + { + var spinner = new SpinWait(); + while (true) + { + var headNow = Volatile.Read(ref _headAndTail.Head); + var tailNow = Volatile.Read(ref _headAndTail.Tail); + + if (headNow == Volatile.Read(ref _headAndTail.Head) && + tailNow == Volatile.Read(ref _headAndTail.Tail)) + { + return GetCount(headNow, tailNow); + } + + spinner.SpinOnce(); + } + } + } + + private int GetCount(int head, int tail) + { + if (head != tail) + { + head &= _mask; + tail &= _mask; + + return head < tail ? tail - head : _buffer.Length - head + tail; + } + return 0; + } + + /// + /// Tries to add the specified item. + /// + /// The item to be added. + /// A BufferStatus value indicating whether the operation succeeded. + /// + /// Thread safe. + /// + public BufferStatus TryAdd(T item) + { + int head = Volatile.Read(ref _headAndTail.Head); + int tail = _headAndTail.Tail; + int size = tail - head; + + if (size >= _buffer.Length) + { + return BufferStatus.Full; + } + + if (Interlocked.CompareExchange(ref _headAndTail.Tail, tail + 1, tail) == tail) + { + int index = tail & _mask; + Volatile.Write(ref _buffer[index], item); + + return BufferStatus.Success; + } + + return BufferStatus.Contended; + } + + + /// + /// Tries to remove an item. + /// + /// The item to be removed. + /// A BufferStatus value indicating whether the operation succeeded. + /// + /// Thread safe for single try take/drain + multiple try add. + /// + public BufferStatus TryTake(out T item) + { + int head = Volatile.Read(ref _headAndTail.Head); + int tail = _headAndTail.Tail; + int size = tail - head; + + if (size == 0) + { + item = default; + return BufferStatus.Empty; + } + + int index = head & _mask; + + item = Volatile.Read(ref _buffer[index]); + + if (item == null) + { + // not published yet + return BufferStatus.Contended; + } + + _buffer[index] = null; + Volatile.Write(ref _headAndTail.Head, ++head); + return BufferStatus.Success; + } + + /// + /// Drains the buffer into the specified array segment. + /// + /// The output buffer + /// The number of items written to the output buffer. + /// + /// Thread safe for single try take/drain + multiple try add. + /// + public int DrainTo(ArraySegment output) => DrainTo(output.AsSpan()); + + /// + /// Drains the buffer into the specified span. + /// + /// The output buffer + /// The number of items written to the output buffer. + /// + /// Thread safe for single try take/drain + multiple try add. + /// + public int DrainTo(Span output) => DrainToImpl(output); + + // use an outer wrapper method to force the JIT to inline the inner adaptor methods + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int DrainToImpl(Span output) + { + int head = Volatile.Read(ref _headAndTail.Head); + int tail = _headAndTail.Tail; + int size = tail - head; + + if (size == 0) + { + return 0; + } + + var localBuffer = _buffer.AsSpan(); + + int outCount = 0; + + do + { + int index = head & _mask; + + T item = Volatile.Read(ref localBuffer[index]); + + if (item == null) + { + // not published yet + break; + } + + localBuffer[index] = null; + Write(output, outCount++, item); + head++; + } + while (head != tail && outCount < Length(output)); + + _headAndTail.Head = head; + + return outCount; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void Write(Span output, int index, T item) => output[index] = item; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int Length(Span output) => output.Length; + + /// + /// Removes all values from the buffer. + /// + /// + /// Not thread safe. + /// + public void Clear() + { + _buffer = new T[_buffer.Length]; + _headAndTail = new PaddedHeadAndTail(); + } +} + +/// +/// Specifies the status of buffer operations. +/// +internal enum BufferStatus +{ + /// + /// The buffer is full. + /// + Full, + + /// + /// The buffer is empty. + /// + Empty, + + /// + /// The buffer operation succeeded. + /// + Success, + + /// + /// The buffer operation was contended. + /// + Contended, +} + +/// +/// Provides utility methods for bit-twiddling operations. +/// +internal static class BitOps +{ + /// + /// Calculate the smallest power of 2 greater than the input parameter. + /// + /// The input parameter. + /// Smallest power of two greater than or equal to x. + public static int CeilingPowerOfTwo(int x) => (int)CeilingPowerOfTwo((uint)x); + + /// + /// Calculate the smallest power of 2 greater than the input parameter. + /// + /// The input parameter. + /// Smallest power of two greater than or equal to x. + public static uint CeilingPowerOfTwo(uint x) => 1u << -BitOperations.LeadingZeroCount(x - 1); + + /// + /// Computes Stafford variant 13 of 64-bit mix function. + /// + /// The input parameter. + /// A bit mix of the input parameter. + /// + /// See http://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html + /// + public static ulong Mix64(ulong z) + { + z = (z ^ z >> 30) * 0xbf58476d1ce4e5b9L; + z = (z ^ z >> 27) * 0x94d049bb133111ebL; + return z ^ z >> 31; + } +} + +[DebuggerDisplay("Head = {Head}, Tail = {Tail}")] +[StructLayout(LayoutKind.Explicit, Size = 3 * Padding.CACHE_LINE_SIZE)] // padding before/between/after fields +internal struct PaddedHeadAndTail +{ + [FieldOffset(1 * Padding.CACHE_LINE_SIZE)] public int Head; + [FieldOffset(2 * Padding.CACHE_LINE_SIZE)] public int Tail; +} + +internal class Padding +{ +#if TARGET_ARM64 || TARGET_LOONGARCH64 + internal const int CACHE_LINE_SIZE = 128; +#else + internal const int CACHE_LINE_SIZE = 64; +#endif +} + diff --git a/test/Benchmarks/Ping/FanoutBenchmark.cs b/test/Benchmarks/Ping/FanoutBenchmark.cs new file mode 100644 index 0000000000..29e1e0f448 --- /dev/null +++ b/test/Benchmarks/Ping/FanoutBenchmark.cs @@ -0,0 +1,150 @@ +using System.Net; +using BenchmarkDotNet.Attributes; +using BenchmarkGrainInterfaces.Ping; +using BenchmarkGrains.Ping; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; +using Orleans.Configuration; + +namespace Benchmarks.Ping +{ + [MemoryDiagnoser] + public class FanoutBenchmark : IDisposable + { + private readonly ConsoleCancelEventHandler _onCancelEvent; + private readonly List hosts = new(); + private readonly ITreeGrain grain; + private readonly IClusterClient client; + private readonly IHost clientHost; + + public FanoutBenchmark() : this(2, true) { } + + public FanoutBenchmark(int numSilos, bool startClient, bool grainsOnSecondariesOnly = false) + { + for (var i = 0; i < numSilos; ++i) + { + var primary = i == 0 ? null : new IPEndPoint(IPAddress.Loopback, 11111); + var hostBuilder = new HostBuilder().UseOrleans((ctx, siloBuilder) => + { +#pragma warning disable ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + siloBuilder.AddActiveRebalancing(); +#pragma warning restore ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + siloBuilder.ConfigureLogging(l => + { + l.AddSimpleConsole(o => + { + o.UseUtcTimestamp = true; + o.TimestampFormat = "HH:mm:ss "; + o.ColorBehavior = LoggerColorBehavior.Enabled; + }); + l.AddFilter("Orleans.Runtime.Placement.Rebalancing", LogLevel.Debug); + }); + siloBuilder.Configure(o => + { + }); + siloBuilder.UseLocalhostClustering( + siloPort: 11111 + i, + gatewayPort: 30000 + i, + primarySiloEndpoint: primary); + + if (i == 0 && grainsOnSecondariesOnly) + { + siloBuilder.Configure(options => options.Classes.Remove(typeof(PingGrain))); + } + }); + + var host = hostBuilder.Build(); + + host.StartAsync().GetAwaiter().GetResult(); + this.hosts.Add(host); + } + + if (grainsOnSecondariesOnly) Thread.Sleep(4000); + + if (startClient) + { + var hostBuilder = new HostBuilder().UseOrleansClient((ctx, clientBuilder) => + { + if (numSilos == 1) + { + clientBuilder.UseLocalhostClustering(); + } + else + { + var gateways = Enumerable.Range(30000, numSilos).Select(i => new IPEndPoint(IPAddress.Loopback, i)).ToArray(); + clientBuilder.UseStaticClustering(gateways); + } + }); + + this.clientHost = hostBuilder.Build(); + this.clientHost.StartAsync().GetAwaiter().GetResult(); + + this.client = this.clientHost.Services.GetRequiredService(); + var grainFactory = this.client; + + this.grain = grainFactory.GetGrain(0, keyExtension: "0"); + this.grain.Ping().AsTask().GetAwaiter().GetResult(); + } + + _onCancelEvent = CancelPressed; + Console.CancelKeyPress += _onCancelEvent; + } + + private void CancelPressed(object sender, ConsoleCancelEventArgs e) + { + Environment.Exit(0); + } + + [Benchmark] + public ValueTask Ping() => grain.Ping(); + + public async Task PingForever() + { + while (true) + { + await grain.Ping(); + } + } + + public async Task Shutdown() + { + if (clientHost is { } client) + { + await client.StopAsync(); + if (client is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + client.Dispose(); + } + } + + this.hosts.Reverse(); + foreach (var host in this.hosts) + { + await host.StopAsync(); + if (host is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + host.Dispose(); + } + } + } + + [GlobalCleanup] + public void Dispose() + { + (this.client as IDisposable)?.Dispose(); + this.hosts.ForEach(h => h.Dispose()); + + Console.CancelKeyPress -= _onCancelEvent; + } + } +} diff --git a/test/Benchmarks/Ping/PingBenchmark.cs b/test/Benchmarks/Ping/PingBenchmark.cs index 34ffc24309..47212eab2f 100644 --- a/test/Benchmarks/Ping/PingBenchmark.cs +++ b/test/Benchmarks/Ping/PingBenchmark.cs @@ -4,6 +4,7 @@ using BenchmarkGrains.Ping; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Orleans.Configuration; namespace Benchmarks.Ping @@ -26,6 +27,17 @@ public PingBenchmark(int numSilos, bool startClient, bool grainsOnSecondariesOnl var primary = i == 0 ? null : new IPEndPoint(IPAddress.Loopback, 11111); var hostBuilder = new HostBuilder().UseOrleans((ctx, siloBuilder) => { +#pragma warning disable ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + siloBuilder.AddActiveRebalancing(); +#pragma warning restore ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + siloBuilder.ConfigureLogging(l => + { + l.AddConsole(); + l.AddFilter("Orleans.Runtime.Placement.Rebalancing", LogLevel.Debug); + }); + siloBuilder.Configure(o => + { + }); siloBuilder.UseLocalhostClustering( siloPort: 11111 + i, gatewayPort: 30000 + i, diff --git a/test/Benchmarks/Ping/TreeGrain.cs b/test/Benchmarks/Ping/TreeGrain.cs new file mode 100644 index 0000000000..985e08ab04 --- /dev/null +++ b/test/Benchmarks/Ping/TreeGrain.cs @@ -0,0 +1,41 @@ +using BenchmarkGrainInterfaces.Ping; + +namespace Benchmarks.Ping; + +public class TreeGrain : Grain, ITreeGrain +{ + // 16^4 grains (~65K) + public const int FanOutFactor = 16; + public const int MaxLevel = 4; + private readonly List _children; + + public TreeGrain() + { + var id = this.GetPrimaryKeyLong(out var forestName); + + var level = id == 0 ? 0 : (int)Math.Log(id, FanOutFactor); + var numChildren = level < MaxLevel ? FanOutFactor : 0; + _children = new List(numChildren); + var childBase = (id + 1) * FanOutFactor; + for (var i = 1; i <= numChildren; i++) + { + var child = GrainFactory.GetGrain(childBase + i, keyExtension: forestName); + _children.Add(child); + } + } + + public async ValueTask Ping() + { + var tasks = new List(_children.Count); + foreach (var child in _children) + { + tasks.Add(child.Ping()); + } + + // Wait for the tasks to complete. + foreach (var task in tasks) + { + await task; + } + } +} diff --git a/test/Benchmarks/Program.cs b/test/Benchmarks/Program.cs index f47c927a5d..d553934471 100644 --- a/test/Benchmarks/Program.cs +++ b/test/Benchmarks/Program.cs @@ -139,6 +139,12 @@ internal class Program { new PingBenchmark(numSilos: 2, startClient: true).PingConcurrent().GetAwaiter().GetResult(); }, + ["ConcurrentPing_TwoSilos_Forever"] = _ => + { + Console.WriteLine("## Client to 2 Silos ##"); + var test = new PingBenchmark(numSilos: 2, startClient: true); + test.PingConcurrentForever().GetAwaiter().GetResult(); + }, ["ConcurrentPing_HostedClient"] = _ => { new PingBenchmark(numSilos: 1, startClient: false).PingConcurrentHostedClient().GetAwaiter().GetResult(); @@ -205,6 +211,10 @@ internal class Program ThreadPool.SetMaxThreads(1, 1); new PingBenchmark().PingForever().GetAwaiter().GetResult(); }, + ["FanoutForever"] = _ => + { + new FanoutBenchmark().PingForever().GetAwaiter().GetResult(); + }, ["GrainStorage.Memory"] = _ => { RunBenchmark( diff --git a/test/Benchmarks/Properties/launchSettings.json b/test/Benchmarks/Properties/launchSettings.json index 82bc182482..58217d2419 100644 --- a/test/Benchmarks/Properties/launchSettings.json +++ b/test/Benchmarks/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "Benchmarks": { "commandName": "Project", - "commandLineArgs": "ConcurrentPing_SiloToSilo_Forever" + "commandLineArgs": "FanoutForever" } } } \ No newline at end of file diff --git a/test/Benchmarks/TopK/TopKBenchmark.cs b/test/Benchmarks/TopK/TopKBenchmark.cs new file mode 100644 index 0000000000..8f6af29783 --- /dev/null +++ b/test/Benchmarks/TopK/TopKBenchmark.cs @@ -0,0 +1,391 @@ +using System.Diagnostics; +using System.Net; +using System.Runtime.CompilerServices; +using BenchmarkDotNet.Attributes; +using Orleans.Placement.Rebalancing; +using Orleans.Runtime; +using Orleans.Runtime.Placement.Rebalancing; + +namespace Benchmarks.TopK; + +[MemoryDiagnoser] +public class TopKBenchmark +{ + private ZipfRejectionSampler _sampler; + private ulong[] ULongSamples; + private Edge[] EdgeSamples; + private EdgeClass[] EdgeClassSamples; + private UlongFrequentItemCollection _fss; + private EdgeClassFrequentItemCollection _fssClass; + private EdgeFrequentItemCollection _fssEdge; + private FrequencySink _sink; + + [Params(100_000, Priority = 3)] + public int Pop { get; set; } + + [Params(0.2, 0.4, 0.6, 0.8, 1.02, 1.2, 1.4, 1.6, Priority = 2)] + public double Skew { get; set; } + + [Params(10_000, Priority = 1)] + public int Cap { get; set; } + + [Params(1_000_000, Priority = 4)] + public int Samples { get; set; } + + [GlobalSetup] + public void GlobalSetup() + { + _sampler = new(new Random(42), Pop, Skew); + + var silos = new SiloAddress[100]; + for (var i = 0; i < silos.Length; i++) + { + silos[i] = SiloAddress.New(new IPEndPoint(IPAddress.Loopback, i), i); + } + + var grains = new GrainId[Pop]; + for (var i = 0; i < Pop; i++) + { + grains[i] = GrainId.Create("grain", i.ToString()); + } + + var grainEdges = new Edge[Pop]; + for (var i = 0; i < Pop; i++) + { + grainEdges[i] = new Edge(new(grains[i % grains.Length], silos[i % silos.Length], true), new(grains[(i + 1) % grains.Length], silos[(i + 1) % silos.Length], true)); + } + + var grainEdgeClasses = new EdgeClass[Pop]; + for (var i = 0; i < Pop; i++) + { + grainEdgeClasses[i] = new(grainEdges[i]); + } + + ULongSamples = new ulong[Samples]; + EdgeSamples = new Edge[Samples]; + EdgeClassSamples = new EdgeClass[Samples]; + for (var i = 0; i < Samples; i++) + { + var sample = _sampler.Sample(); + ULongSamples[i] = (ulong)sample; + EdgeSamples[i] = grainEdges[sample % grainEdges.Length]; + EdgeClassSamples[i] = grainEdgeClasses[sample % grainEdgeClasses.Length]; + } + + _fss = new UlongFrequentItemCollection(Cap); + _fssClass = new EdgeClassFrequentItemCollection(Cap); + _fssEdge = new EdgeFrequentItemCollection(Cap); + _sink = new FrequencySink(Cap); + } + + internal sealed record class EdgeClass(Edge Edge); + + [IterationSetup] + public void IterationSetup() + { + /* + _fss.Clear(); + _fssEdge.Clear(); + _fssClass.Clear(); + */ + //_sink = new FrequencySink(Cap); + } + + /* + [Benchmark] + [BenchmarkCategory("Add")] + public void FssULongAdd() + { + foreach (var sample in ULongSamples) + { + _fss.Add(sample); + } + } + */ + + /* + [Benchmark] + [BenchmarkCategory("Add")] + public void FssClassAdd() + { + foreach (var sample in EdgeClassSamples) + { + _fssClass.Add(sample); + } + } + */ + + [Benchmark] + [BenchmarkCategory("FSS")] + public void FssAdd() + { + foreach (var sample in EdgeSamples) + { + _fssEdge.Add(sample); + } + } + + [Benchmark] + [BenchmarkCategory("SS")] + public void SinkAdd() + { + foreach (var sample in EdgeSamples) + { + _sink.Add(sample); + } + } + + private sealed class EdgeFrequentItemCollection(int capacity) : FrequentItemCollection(capacity) + { + protected override ulong GetKey(in Edge element) => (ulong)element.Source.Id.GetUniformHashCode() << 32 | element.Target.Id.GetUniformHashCode(); + public void Clear() => ClearCore(); + } + + private sealed class EdgeClassFrequentItemCollection(int capacity) : FrequentItemCollection(capacity) + { + static ulong GetKey(in Edge element) => (ulong)element.Source.Id.GetUniformHashCode() << 32 | element.Target.Id.GetUniformHashCode(); + protected override ulong GetKey(in EdgeClass element) => GetKey(element.Edge); + public void Clear() => ClearCore(); + } + + private sealed class UlongFrequentItemCollection(int capacity) : FrequentItemCollection(capacity) + { + protected override ulong GetKey(in ulong element) => element; + public void Remove(in ulong element) => Remove(GetKey(element)); + public void Clear() => ClearCore(); + } + + // https://jasoncrease.medium.com/rejection-sampling-the-zipf-distribution-6b359792cffa + public class ZipfRejectionSampler + { + private readonly Random _rand; + private readonly double _skew; + private readonly double _t; + + public ZipfRejectionSampler(Random random, long cardinality, double skew) + { + _rand = random; + _skew = skew; + _t = (Math.Pow(cardinality, 1 - skew) - skew) / (1 - skew); + } + + public long Sample() + { + while (true) + { + double invB = bInvCdf(_rand.NextDouble()); + long sampleX = (long)(invB + 1); + double yRand = _rand.NextDouble(); + double ratioTop = Math.Pow(sampleX, -_skew); + double ratioBottom = sampleX <= 1 ? 1 / _t : Math.Pow(invB, -_skew) / _t; + double rat = (ratioTop) / (ratioBottom * _t); + + if (yRand < rat) + return sampleX; + } + } + private double bInvCdf(double p) + { + if (p * _t <= 1) + return p * _t; + else + return Math.Pow((p * _t) * (1 - _skew) + _skew, 1 / (1 - _skew)); + } + } + + + internal class EdgeCounter(ulong value, Edge edge) + { + public ulong Value { get; set; } = value; + public Edge Edge { get; } = edge; + + } + +/// +/// Implementation of the Space-Saving algorithm: https://www.cse.ust.hk/~raywong/comp5331/References/EfficientComputationOfFrequentAndTop-kElementsInDataStreams.pdf +/// +internal sealed class FrequencySink(int capacity) +{ + public ulong GetKey(in Edge element) => (ulong)element.Source.Id.GetUniformHashCode() << 32 | element.Target.Id.GetUniformHashCode(); + private readonly Dictionary _counters = new(capacity); + private readonly UpdateableMinHeap _heap = new(capacity); + + public int Capacity { get; } = capacity; + public Dictionary.ValueCollection Counters => _counters.Values; + + public void Add(Edge edge) + { + var combinedHash = GetKey(edge); + if (_counters.TryGetValue(combinedHash, out var counter)) + { + counter.Value++; + _heap.Update(combinedHash, counter.Value); + + return; + } + + if (_counters.Count == Capacity) + { + var minHash = _heap.Dequeue(); + _counters.Remove(minHash); + } + + _counters.Add(combinedHash, new EdgeCounter(1, edge)); + _heap.Enqueue(combinedHash, _counters[combinedHash].Value); + } + + public void Remove(uint sourceHash, uint targetHash) + { + var combinedHash = CombineHashes(sourceHash, targetHash); + var reversedHash = CombineHashes(targetHash, sourceHash); + + if (_counters.Remove(combinedHash)) + { + _ = _heap.Remove(combinedHash); + } + + if (_counters.Remove(reversedHash)) + { + _ = _heap.Remove(reversedHash); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong CombineHashes(uint sourceHash, uint targetHash) + => (ulong)sourceHash << 32 | targetHash; + + // Inspired by: https://github.com/DesignEngrLab/TVGL/blob/master/TessellationAndVoxelizationGeometryLibrary/Miscellaneous%20Functions/UpdatablePriorityQueue.cs + private class UpdateableMinHeap(int capacity) + { + private const int Arity = 4; + private const int Log2Arity = 2; + + private readonly Dictionary _hashIndexes = new(capacity); + private readonly (ulong Hash, ulong Value)[] _nodes = new (ulong, ulong)[capacity]; + + private int _size; + + public void Enqueue(ulong hash, ulong value) + { + var currentSize = _size; + _size = currentSize + 1; + + MoveNodeUp((hash, value), currentSize); + } + + public ulong Dequeue() + { + var hash = _nodes[0].Hash; + _hashIndexes.Remove(hash); + + var lastNodeIndex = --_size; + if (lastNodeIndex > 0) + { + var lastNode = _nodes[lastNodeIndex]; + MoveNodeDown(lastNode, 0); + } + + return hash; + } + + public bool Remove(ulong hash) + { + if (!_hashIndexes.TryGetValue(hash, out var index)) + { + return false; + } + + var nodes = _nodes; + var newSize = --_size; + + if (index < newSize) + { + var lastNode = nodes[newSize]; + MoveNodeDown(lastNode, index); + } + + _hashIndexes.Remove(hash); + nodes[newSize] = default; + + return true; + } + + public void Update(ulong hash, ulong newValue) + { + Remove(hash); + Enqueue(hash, newValue); + } + + private void MoveNodeUp((ulong Hash, ulong Value) node, int nodeIndex) + { + Debug.Assert(0 <= nodeIndex && nodeIndex < _size); + + var nodes = _nodes; + + while (nodeIndex > 0) + { + var parentIndex = GetParentIndex(nodeIndex); + var parent = nodes[parentIndex]; + + if (Comparer.Default.Compare(node.Value, parent.Value) < 0) + { + nodes[nodeIndex] = parent; + _hashIndexes[parent.Hash] = nodeIndex; + nodeIndex = parentIndex; + } + else + { + break; + } + } + + _hashIndexes[node.Hash] = nodeIndex; + nodes[nodeIndex] = node; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int GetParentIndex(int index) => (index - 1) >> Log2Arity; + } + + private void MoveNodeDown((ulong Hash, ulong Value) node, int nodeIndex) + { + Debug.Assert(0 <= nodeIndex && nodeIndex < _size); + + var nodes = _nodes; + var size = _size; + + int i; + while ((i = GetFirstChildIndex(nodeIndex)) < size) + { + var minChild = nodes[i]; + var minChildIndex = i; + + var childIndexUpperBound = Math.Min(i + Arity, size); + while (++i < childIndexUpperBound) + { + var nextChild = nodes[i]; + if (nextChild.Value < minChild.Value) + { + minChild = nextChild; + minChildIndex = i; + } + } + + if (node.Value <= minChild.Value) + { + break; + } + + nodes[nodeIndex] = minChild; + _hashIndexes[minChild.Hash] = nodeIndex; + nodeIndex = minChildIndex; + } + + _hashIndexes[node.Hash] = nodeIndex; + nodes[nodeIndex] = node; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int GetFirstChildIndex(int index) => (index << Log2Arity) + 1; + } + } +} +} diff --git a/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/ConcurrentLoadGenerator.cs b/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/ConcurrentLoadGenerator.cs index 5c51cc5bbc..dab5ecf0a9 100644 --- a/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/ConcurrentLoadGenerator.cs +++ b/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/ConcurrentLoadGenerator.cs @@ -1,6 +1,7 @@ using System.Threading.Channels; using System.Diagnostics; using Microsoft.Extensions.Logging; +using Microsoft.Crank.EventSources; namespace DistributedTests.Client { @@ -126,6 +127,11 @@ public async Task Run(CancellationToken ct) if (!more) break; while (completedBlockReader.TryRead(out var block)) { + // Register the measurement values + BenchmarksEventSource.Measure("requests", block.Completed); + BenchmarksEventSource.Measure("failures", block.Failures); + BenchmarksEventSource.Measure("rps", block.RequestsPerSecond); + blocks.Add(block); } diff --git a/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarioRunner.cs b/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarioRunner.cs index 475020d26c..761534e936 100644 --- a/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarioRunner.cs +++ b/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarioRunner.cs @@ -35,6 +35,11 @@ public LoadGeneratorScenarioRunner(ILoadGeneratorScenario scenario, ILoggerFa public async Task Run(ClientParameters clientParams, LoadGeneratorParameters loadParams) { + // Register the measurements. n0 -> format as natural number + BenchmarksEventSource.Register("requests", Operations.Sum, Operations.Sum, "Requests", "Number of requests completed", "n0"); + BenchmarksEventSource.Register("failures", Operations.Sum, Operations.Sum, "Failures", "Number of failures", "n0"); + BenchmarksEventSource.Register("rps", Operations.Sum, Operations.Median, "Median RPS", "Rate per second", "n0"); + var secrets = SecretConfiguration.Load(clientParams.SecretSource); var hostBuilder = new HostBuilder().UseOrleansClient((ctx, builder) => builder.Configure(options => { options.ClusterId = clientParams.ClusterId; options.ServiceId = clientParams.ServiceId; }) @@ -65,15 +70,8 @@ public async Task Run(ClientParameters clientParams, LoadGeneratorParameters loa _logger.LogInformation("Running"); var report = await generator.Run(cts.Token); - // Register the measurements. n0 -> format as natural number - BenchmarksEventSource.Register("requests", Operations.First, Operations.Sum, "Requests", "Number of requests completed", "n0"); - BenchmarksEventSource.Register("failures", Operations.First, Operations.Sum, "Failures", "Number of failures", "n0"); - BenchmarksEventSource.Register("rps", Operations.First, Operations.Sum, "Rate per second", "Rate per seconds", "n0"); - - // Register the measurement values - BenchmarksEventSource.Measure("requests", report.Completed); - BenchmarksEventSource.Measure("failures", report.Failures); - BenchmarksEventSource.Measure("rps", report.RatePerSecond); + BenchmarksEventSource.Register("overall-rps", Operations.Last, Operations.Last, "Overall RPS", "RPS", "n0"); + BenchmarksEventSource.Measure("overall-rps", report.RatePerSecond); await host.StopAsync(); } diff --git a/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarios.cs b/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarios.cs index 18230bdf45..c0ad25980f 100644 --- a/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarios.cs +++ b/test/DistributedTests/DistributedTests.Client/LoadGeneratorScenario/LoadGeneratorScenarios.cs @@ -19,4 +19,13 @@ public class PingScenario : ILoadGeneratorScenario public ValueTask IssueRequest(IPingGrain state) => state.Ping(); } + + public class FanOutScenario : ILoadGeneratorScenario + { + public string Name => "fan-out"; + + public ITreeGrain GetStateForWorker(IClusterClient client, int workerId) => client.GetGrain(primaryKey: 0, keyExtension: workerId.ToString()); + + public ValueTask IssueRequest(ITreeGrain root) => root.Ping(); + } } diff --git a/test/DistributedTests/DistributedTests.Client/Program.cs b/test/DistributedTests/DistributedTests.Client/Program.cs index 0d7086d792..93ecce55b2 100644 --- a/test/DistributedTests/DistributedTests.Client/Program.cs +++ b/test/DistributedTests/DistributedTests.Client/Program.cs @@ -10,6 +10,7 @@ var root = new RootCommand(); root.Add(Scenario.CreateCommand(new PingScenario(), loggerFactory)); +root.Add(Scenario.CreateCommand(new FanOutScenario(), loggerFactory)); root.Add(new CounterCaptureCommand(loggerFactory.CreateLogger())); root.Add(new ChaosAgentCommand(loggerFactory.CreateLogger())); diff --git a/test/DistributedTests/DistributedTests.Common/GrainInterfaces/IPingGrain.cs b/test/DistributedTests/DistributedTests.Common/GrainInterfaces/IPingGrain.cs index fbb1e1e633..f49b3e2f82 100644 --- a/test/DistributedTests/DistributedTests.Common/GrainInterfaces/IPingGrain.cs +++ b/test/DistributedTests/DistributedTests.Common/GrainInterfaces/IPingGrain.cs @@ -1,7 +1,7 @@ -namespace DistributedTests.GrainInterfaces +namespace DistributedTests.GrainInterfaces; + +public interface IPingGrain : IGrainWithGuidKey { - public interface IPingGrain : IGrainWithGuidKey - { - ValueTask Ping(); - } + ValueTask Ping(); } + diff --git a/test/DistributedTests/DistributedTests.Common/GrainInterfaces/ITreeGrain.cs b/test/DistributedTests/DistributedTests.Common/GrainInterfaces/ITreeGrain.cs new file mode 100644 index 0000000000..d67ff34b8d --- /dev/null +++ b/test/DistributedTests/DistributedTests.Common/GrainInterfaces/ITreeGrain.cs @@ -0,0 +1,7 @@ +namespace DistributedTests.GrainInterfaces; + +public interface ITreeGrain : IGrainWithIntegerCompoundKey +{ + public ValueTask Ping(); +} + diff --git a/test/DistributedTests/DistributedTests.Grains/PingGrain.cs b/test/DistributedTests/DistributedTests.Grains/PingGrain.cs index 0d00061df9..4e01b65693 100644 --- a/test/DistributedTests/DistributedTests.Grains/PingGrain.cs +++ b/test/DistributedTests/DistributedTests.Grains/PingGrain.cs @@ -1,9 +1,8 @@ using DistributedTests.GrainInterfaces; -namespace DistributedTests.Grains +namespace DistributedTests.Grains; + +public class PingGrain : Grain, IPingGrain { - public class PingGrain : Grain, IPingGrain - { - public ValueTask Ping() => default; - } + public ValueTask Ping() => default; } diff --git a/test/DistributedTests/DistributedTests.Grains/TreeGrain.cs b/test/DistributedTests/DistributedTests.Grains/TreeGrain.cs new file mode 100644 index 0000000000..b5c8817455 --- /dev/null +++ b/test/DistributedTests/DistributedTests.Grains/TreeGrain.cs @@ -0,0 +1,41 @@ +using DistributedTests.GrainInterfaces; + +namespace DistributedTests.Grains; + +public class TreeGrain : Grain, ITreeGrain +{ + // 16^4 grains (~65K) + public const int FanOutFactor = 16; + public const int MaxLevel = 4; + private readonly List _children; + + public TreeGrain() + { + var id = this.GetPrimaryKeyLong(out var forestName); + + var level = id == 0 ? 0 : (int)Math.Log(id, FanOutFactor); + var numChildren = level < MaxLevel ? FanOutFactor : 0; + _children = new List(numChildren); + var childBase = (id + 1) * FanOutFactor; + for (var i = 1; i <= numChildren; i++) + { + var child = GrainFactory.GetGrain(childBase + i, keyExtension: forestName); + _children.Add(child); + } + } + + public async ValueTask Ping() + { + var tasks = new List(_children.Count); + foreach (var child in _children) + { + tasks.Add(child.Ping()); + } + + // Wait for the tasks to complete. + foreach (var task in tasks) + { + await task; + } + } +} diff --git a/test/DistributedTests/DistributedTests.Server/ServerCommand.cs b/test/DistributedTests/DistributedTests.Server/ServerCommand.cs index b1d0e67a5d..14944aa8e1 100644 --- a/test/DistributedTests/DistributedTests.Server/ServerCommand.cs +++ b/test/DistributedTests/DistributedTests.Server/ServerCommand.cs @@ -19,6 +19,7 @@ public ServerCommand(ISiloConfigurator siloConfigurator) AddOption(OptionHelper.CreateOption("--siloPort", defaultValue: 11111)); AddOption(OptionHelper.CreateOption("--gatewayPort", defaultValue: 30000)); AddOption(OptionHelper.CreateOption("--secretSource", defaultValue: SecretConfiguration.SecretSource.File)); + AddOption(OptionHelper.CreateOption("--activeRebalancing", defaultValue: true)); foreach (var opt in siloConfigurator.Options) { diff --git a/test/DistributedTests/DistributedTests.Server/ServerRunner.cs b/test/DistributedTests/DistributedTests.Server/ServerRunner.cs index ec3588755b..3ac709ace7 100644 --- a/test/DistributedTests/DistributedTests.Server/ServerRunner.cs +++ b/test/DistributedTests/DistributedTests.Server/ServerRunner.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Hosting; using Orleans.Configuration; using DistributedTests.Common.MessageChannel; +using Microsoft.Extensions.Logging; namespace DistributedTests.Server { @@ -12,6 +13,7 @@ public class CommonParameters public int SiloPort { get; set; } public int GatewayPort { get; set; } public SecretConfiguration.SecretSource SecretSource { get; set; } + public bool ActiveRebalancing { get; set; } } public class ServerRunner @@ -38,6 +40,10 @@ public async Task Run(CommonParameters commonParameters, T configuratorParameter { var host = Host .CreateDefaultBuilder() + .ConfigureLogging(logging => + { + logging.AddFilter("Orleans.Runtime.Placement.Rebalancing", LogLevel.Debug); + }) .UseOrleans((ctx, siloBuilder) => ConfigureOrleans(siloBuilder, commonParameters, configuratorParameters)) .Build(); @@ -70,6 +76,13 @@ private void ConfigureOrleans(ISiloBuilder siloBuilder, CommonParameters commonP .ConfigureEndpoints(siloPort: commonParameters.SiloPort, gatewayPort: commonParameters.GatewayPort) .UseAzureStorageClustering(options => options.TableServiceClient = new(_secrets.ClusteringConnectionString)); + if (commonParameters.ActiveRebalancing) + { +#pragma warning disable ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + siloBuilder.AddActiveRebalancing(); +#pragma warning restore ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + _siloConfigurator.Configure(siloBuilder, configuratorParameters); } } diff --git a/test/Grains/BenchmarkGrainInterfaces/Ping/ITreeGrain.cs b/test/Grains/BenchmarkGrainInterfaces/Ping/ITreeGrain.cs new file mode 100644 index 0000000000..75540f8508 --- /dev/null +++ b/test/Grains/BenchmarkGrainInterfaces/Ping/ITreeGrain.cs @@ -0,0 +1,7 @@ +namespace BenchmarkGrainInterfaces.Ping; + +public interface ITreeGrain : IGrainWithIntegerCompoundKey +{ + public ValueTask Ping(); +} + diff --git a/test/NonSilo.Tests/General/RingTests_Standalone.cs b/test/NonSilo.Tests/General/RingTests_Standalone.cs index ac8de6598a..97fc2738ec 100644 --- a/test/NonSilo.Tests/General/RingTests_Standalone.cs +++ b/test/NonSilo.Tests/General/RingTests_Standalone.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using System.Net; using Microsoft.Extensions.Logging.Abstractions; using Orleans.Runtime; @@ -259,6 +260,7 @@ public bool TryGetSiloName(SiloAddress siloAddress, out string siloName) } public bool UnSubscribeFromSiloStatusEvents(ISiloStatusListener observer) => _subscribers.Remove(observer); + public ImmutableArray GetActiveSilos() => [.. GetApproximateSiloStatuses(onlyActive: true).Keys]; } internal class RangeBreakable diff --git a/test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs b/test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs new file mode 100644 index 0000000000..bf35feeb31 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs @@ -0,0 +1,47 @@ +using Orleans.Placement.Rebalancing; +using Orleans.Runtime.Placement.Rebalancing; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +public sealed class CandidateVertexMaxHeapTests +{ + [Fact] + void HeapPropertyIsMaintained() + { + var edges = new CandidateVertex[100]; + for (int i = 0; i < edges.Length; i++) + { + edges[i] = new CandidateVertex { AccumulatedTransferScore = i }; + } + + Random.Shared.Shuffle(edges); + var heap = new CandidateVertexMaxHeap(edges); + Assert.Equal(100, heap.Count); + Assert.Equal(99, heap.Peek().AccumulatedTransferScore); + Assert.Equal(99, heap.Peek().AccumulatedTransferScore); + Assert.Equal(99, heap.Pop().AccumulatedTransferScore); + Assert.Equal(98, heap.Pop().AccumulatedTransferScore); + Assert.Equal(98, heap.Count); + Assert.Equal(98, heap.UnorderedElements.Count()); + + // Randomly re-assign priorities to edges + var newScore = 1000; + var elements = heap.UnorderedElements.ToArray(); + Random.Shared.Shuffle(edges); + + foreach (var element in elements) + { + element.AccumulatedTransferScore = newScore--; + } + + heap.Heapify(); + + Assert.Equal(1000, heap.Peek().AccumulatedTransferScore); + Assert.Equal(1000, heap.Peek().AccumulatedTransferScore); + Assert.Equal(1000, heap.Pop().AccumulatedTransferScore); + Assert.Equal(999, heap.Pop().AccumulatedTransferScore); + Assert.Equal(96, heap.Count); + Assert.Equal(96, heap.UnorderedElements.Count()); + } +} diff --git a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs new file mode 100644 index 0000000000..91cd69cbb8 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs @@ -0,0 +1,223 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration; +using Orleans.Placement; +using Orleans.Placement.Rebalancing; +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Rebalancing; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +[TestCategory("Functional"), TestCategory("ActiveRebalancing")] +public class CustomToleranceTests(CustomToleranceTests.Fixture fixture) : RebalancingTestBase(fixture), IClassFixture +{ + [Fact] + public async Task Should_ConvertAllRemoteCalls_ToLocalCalls_WhileRespectingTolerance() + { + await AdjustActivationCountOffsets(); + + var e1 = GrainFactory.GetGrain(1); + var e2 = GrainFactory.GetGrain(2); + var e3 = GrainFactory.GetGrain(3); + + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo1); + + await e1.FirstPing(Silo2); + await e2.FirstPing(Silo2); + await e3.FirstPing(Silo2); + + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo2); + var x = GrainFactory.GetGrain(0); + await x.Ping(); + + var i = 0; + while (i < 3) + { + await e1.Ping(); + await e2.Ping(); + await e3.Ping(); + await x.Ping(); + i++; + } + + var f1 = GrainFactory.GetGrain(1); + var f2 = GrainFactory.GetGrain(2); + var f3 = GrainFactory.GetGrain(3); + + var e1_host = await e1.GetAddress(); + var e2_host = await e2.GetAddress(); + var e3_host = await e3.GetAddress(); + + var f1_host = await f1.GetAddress(); + var f2_host = await f2.GetAddress(); + var f3_host = await f3.GetAddress(); + + Assert.Equal(Silo1, e1_host); + Assert.Equal(Silo1, e2_host); + Assert.Equal(Silo1, e3_host); + + Assert.Equal(Silo2, f1_host); + Assert.Equal(Silo2, f2_host); + Assert.Equal(Silo2, f3_host); + + Assert.Equal(Silo2, await x.GetAddress()); // X remains in silo 2 + + await Silo1Rebalancer.TriggerExchangeRequest(); + + do + { + e2_host = await e2.GetAddress(); + e3_host = await e3.GetAddress(); + f1_host = await f1.GetAddress(); + } + while (e2_host == Silo1 || e3_host == Silo1 || f1_host == Silo2); + + await Test(); + + // At this point the layout is like follows: + + // S1: E1-F1, sys.svc.clustering.dev, rest (default activations, present in both silos) + // S2: E2-F2, E3-F3, X, rest (default activations, present in both silos) + + // Tolerance <= 2, and if we ignore defaults once, sys.svc.clustering.dev, and X (which is used to counter-balance sys.svc.clustering.dev) + // we end up with a total of 2 activations in silo1, and 4 in silo 2, which means the tolerance has been respected, and all remote calls have + // been converted to local calls: S1: E1-F1, S2: E2-F2, s2: E3-F3. + + // To make sure, we trigger 's1_rebalancer' again, which should yield to no further migrations. + i = 0; + while (i < 3) + { + await e1.Ping(); + await e2.Ping(); + await e3.Ping(); + await x.Ping(); + i++; + } + + await Silo1Rebalancer.TriggerExchangeRequest(); + await Test(); + + // To make extra sure, we now trigger 's2_rebalancer', which again should yield to no further migrations. + i = 0; + while (i < 3) + { + await e1.Ping(); + await e2.Ping(); + await e3.Ping(); + await x.Ping(); + i++; + } + + await Silo2Rebalancer.TriggerExchangeRequest(); + await Test(); + + //await ResetCounters(); uncomment if you add more tests + + async Task Test() + { + e1_host = await e1.GetAddress(); + e2_host = await e2.GetAddress(); + e3_host = await e3.GetAddress(); + + f1_host = await f1.GetAddress(); + f2_host = await f2.GetAddress(); + f3_host = await f3.GetAddress(); + + Assert.Equal(Silo1, e1_host); // E1 is still in silo 1 + Assert.Equal(Silo2, e2_host); // E2 is now in silo 2 + Assert.Equal(Silo2, e3_host); // E3 is now in silo 2 + + Assert.Equal(Silo1, f1_host); // F1 is now in silo 1 + Assert.Equal(Silo2, f2_host); // F2 is still in silo 2 + Assert.Equal(Silo2, f3_host); // F3 is still in silo 2 + + Assert.Equal(Silo2, await x.GetAddress()); // X remains in silo 2 + } + } + + public interface IE : IGrainWithIntegerKey + { + Task FirstPing(SiloAddress silo2); + Task Ping(); + Task GetAddress(); + } + public interface IF : IGrainWithIntegerKey + { + Task Ping(); + Task GetAddress(); + } + public interface IX : IGrainWithIntegerKey + { + Task Ping(); + Task GetAddress(); + } + + public class E : Grain, IE + { + public async Task FirstPing(SiloAddress silo2) + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, silo2); + await GrainFactory.GetGrain(this.GetPrimaryKeyLong()).Ping(); + } + + public Task Ping() => GrainFactory.GetGrain(this.GetPrimaryKeyLong()).Ping(); + public Task GetAddress() => Task.FromResult(GrainContext.Address.SiloAddress); + } + + public class F : Grain, IF + { + public Task Ping() => Task.CompletedTask; + public Task GetAddress() => Task.FromResult(GrainContext.Address.SiloAddress); + } + + /// + /// This is simply to achive initial balance between the 2 silos, as by default the primary + /// will have 1 more activation than the secondary. That activations is 'sys.svc.clustering.dev' + /// + [Immovable] + public class X : Grain, IX + { + public Task Ping() => Task.CompletedTask; + public Task GetAddress() => Task.FromResult(GrainContext.Address.SiloAddress); + } + + public class Fixture : BaseTestClusterFixture + { + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.Options.InitialSilosCount = 2; + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) +#pragma warning disable ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + => hostBuilder + .Configure(o => + { + o.AssumeHomogenousSilosForTesting = true; + o.ClientGatewayShutdownNotificationTimeout = default; + }) + .Configure(o => + { + // Make these so that the timers practically never fire! We will invoke the protocol manually. + o.MinRebalancingDueTime = TimeSpan.FromSeconds(299); + o.MaxRebalancingDueTime = TimeSpan.FromSeconds(300); + // Make this practically zero, so we can invoke the protocol more than once without needing to put a delay in the tests. + o.RecoveryPeriod = TimeSpan.FromMilliseconds(1); + }) + .AddActiveRebalancing() + .ConfigureServices(service => service.AddSingleton()); +#pragma warning restore ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + + private class HardLimitRule : IImbalanceToleranceRule + { + public bool IsSatisfiedBy(uint imbalance) => imbalance <= 2; + } + } +} \ No newline at end of file diff --git a/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs new file mode 100644 index 0000000000..7dd8ec5741 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs @@ -0,0 +1,619 @@ +using System.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration; +using Orleans.Placement; +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Rebalancing; +using Orleans.Streams; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +[TestCategory("Functional"), TestCategory("ActiveRebalancing")] +public class DefaultToleranceTests(DefaultToleranceTests.Fixture fixture) : RebalancingTestBase(fixture), IClassFixture +{ + [Fact] + public async Task A_ShouldMoveToSilo2__B_And_C_ShouldStayOnSilo2() + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo1); + + var scenario = Scenario._1; + var a = GrainFactory.GetGrain($"a{scenario}"); + await a.FirstPing(scenario, Silo1, Silo2); + + var i = 0; + while (i < 3) + { + await a.Ping(scenario); + i++; + } + + var b = GrainFactory.GetGrain($"b{scenario}"); + var c = GrainFactory.GetGrain($"c{scenario}"); + + var a_host = await a.GetAddress(); + var b_host = await b.GetAddress(); + var c_host = await c.GetAddress(); + + Assert.Equal(Silo1, a_host); + Assert.Equal(Silo2, b_host); + Assert.Equal(Silo2, c_host); + + await Silo1Rebalancer.TriggerExchangeRequest(); + + do + { + a_host = await a.GetAddress(); + } + while (a_host == Silo1); + + // refresh + a_host = await a.GetAddress(); + b_host = await b.GetAddress(); + c_host = await c.GetAddress(); + + Assert.Equal(Silo2, a_host); // A is now in silo 2 + Assert.Equal(Silo2, b_host); + Assert.Equal(Silo2, c_host); + + await ResetCounters(); + } + + [Fact] + public async Task C_ShouldMoveToSilo1__A_And_B_ShouldStayOnSilo1() + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo1); + + var scenario = Scenario._2; + var a = GrainFactory.GetGrain($"a{scenario}"); + var b = GrainFactory.GetGrain($"b{scenario}"); + + await a.FirstPing(scenario, Silo1, Silo2); + await b.Ping(scenario); + + var i = 0; + while (i < 3) + { + await a.Ping(scenario); + await b.Ping(scenario); + i++; + } + + var c = GrainFactory.GetGrain($"c{scenario}"); + + var a_host = await a.GetAddress(); + var b_host = await b.GetAddress(); + var c_host = await c.GetAddress(); + + Assert.Equal(Silo1, a_host); + Assert.Equal(Silo1, b_host); + Assert.Equal(Silo2, c_host); + + await Silo1Rebalancer.TriggerExchangeRequest(); + + do + { + c_host = await c.GetAddress(); + } + while (c_host == Silo2); + + // refresh + a_host = await a.GetAddress(); + b_host = await b.GetAddress(); + c_host = await c.GetAddress(); + + Assert.Equal(Silo1, a_host); + Assert.Equal(Silo1, b_host); + Assert.Equal(Silo1, c_host); // C is now in silo 1 + + await ResetCounters(); + } + + [Fact] + public async Task Immovable_C_ShouldStayOnSilo2__A_And_B_ShouldMoveToSilo2() + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo1); + + var scenario = Scenario._3; + var a = GrainFactory.GetGrain($"a{scenario}"); + var b = GrainFactory.GetGrain($"b{scenario}"); + + await a.FirstPing(scenario, Silo1, Silo2); + await b.Ping(scenario); + + var i = 0; + while (i < 3) + { + await a.Ping(scenario); + await b.Ping(scenario); + i++; + } + + var c = GrainFactory.GetGrain($"c{scenario}"); + + var a_host = await a.GetAddress(); + var b_host = await b.GetAddress(); + var c_host = await c.GetAddress(); + + Assert.Equal(Silo1, a_host); + Assert.Equal(Silo1, b_host); + Assert.Equal(Silo2, c_host); + + await Silo1Rebalancer.TriggerExchangeRequest(); + + do + { + a_host = await a.GetAddress(); + b_host = await b.GetAddress(); + } + while (a_host == Silo1 || b_host == Silo1); + + // refresh + a_host = await a.GetAddress(); + b_host = await b.GetAddress(); + c_host = await c.GetAddress(); + + Assert.Equal(Silo2, a_host); // A is now in silo 2 + Assert.Equal(Silo2, b_host); // B is now in silo 2 + Assert.Equal(Silo2, c_host); + + await ResetCounters(); + } + + [Fact] + public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOtherWayAround() + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo1); + + var scenario = Scenario._4; + var a = GrainFactory.GetGrain($"a{scenario}"); + + await a.FirstPing(scenario, Silo1, Silo2); + + for (var i = 0; i < 3; ++i) + { + await a.Ping(scenario); + } + + var b = GrainFactory.GetGrain($"b{scenario}"); + var c = GrainFactory.GetGrain($"c{scenario}"); + var d = GrainFactory.GetGrain($"d{scenario}"); + + var a_host = await a.GetAddress(); + var b_host = await b.GetAddress(); + var c_host = await c.GetAddress(); + var d_host = await d.GetAddress(); + + Assert.Equal(Silo1, a_host); + Assert.Equal(Silo1, b_host); + Assert.Equal(Silo2, c_host); + Assert.Equal(Silo2, d_host); + + // 1st cycle + await Silo1Rebalancer.TriggerExchangeRequest(); + + do + { + a_host = await a.GetAddress(); + c_host = await c.GetAddress(); + } + while (a_host != c_host); + + // refresh + a_host = await a.GetAddress(); + b_host = await b.GetAddress(); + c_host = await c.GetAddress(); + d_host = await d.GetAddress(); + + // A can go to Silo 2, or C can come to Silo 1, both are valid, so we need to check for both! + if (a_host == Silo2 && c_host == Silo2) + { + // A is now in silo 2 + Assert.Equal(Silo2, a_host); + Assert.Equal(Silo1, b_host); + Assert.Equal(Silo2, c_host); + Assert.Equal(Silo2, d_host); + + // 2nd cycle + for (var i = 0; i < 3; i++) + { + await a.Ping(scenario); + } + + // Since A moved to silo 2 at this point, it will be twice as strongly connected to C as it is to B, + // even though its now making remote calls (to B)! Thats why we trigger the exchange from 's1_rebalancer' + await Silo1Rebalancer.TriggerExchangeRequest(); + + do + { + b_host = await b.GetAddress(); + } + while (b_host == Silo1); + + // refresh + a_host = await a.GetAddress(); + b_host = await b.GetAddress(); + c_host = await c.GetAddress(); + d_host = await d.GetAddress(); + + Assert.Equal(Silo2, a_host); + Assert.Equal(Silo2, b_host); // B is now in silo 2 + Assert.Equal(Silo2, c_host); + Assert.Equal(Silo2, d_host); + + return; + } + + if (a_host == Silo1 && c_host == Silo1) + { + Assert.Equal(Silo1, a_host); + Assert.Equal(Silo1, b_host); + Assert.Equal(Silo1, c_host); // C is now in silo 1 + Assert.Equal(Silo2, d_host); + + // 2nd cycle + for (var i = 0; i < 3; i++) + { + await a.Ping(scenario); + } + + // Since C moved to silo 1 at this point, it will be twice as strongly connected to A as it is to D, + // even though its now making remote calls (to D)! Thats why we trigger the exchange from 's2_rebalancer' + await Silo2Rebalancer.TriggerExchangeRequest(); + + do + { + d_host = await b.GetAddress(); + } + while (d_host == Silo2); + + // refresh + a_host = await a.GetAddress(); + b_host = await b.GetAddress(); + c_host = await c.GetAddress(); + d_host = await d.GetAddress(); + + Assert.Equal(Silo1, a_host); + Assert.Equal(Silo1, b_host); + Assert.Equal(Silo1, c_host); + Assert.Equal(Silo1, d_host); // D is now in silo 1 + + return; + } + + await ResetCounters(); + } + + [SkippableFact] + public async Task Receivers_ShouldMoveCloseTo_PullingAgent() + { + var sp1 = GrainFactory.GetGrain("s1"); + var sp2 = GrainFactory.GetGrain("s2"); + var sp3 = GrainFactory.GetGrain("s3"); + + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo1); + await sp1.FirstPing(); + await sp2.FirstPing(); + + RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo2); + await sp3.FirstPing(); + + var i = 0; + while (i < 3) + { + await sp1.StreamPing(); + await sp2.StreamPing(); + await sp3.StreamPing(); + i++; + } + + var sr1 = GrainFactory.GetGrain("s1"); + var sr2 = GrainFactory.GetGrain("s2"); + var sr3 = GrainFactory.GetGrain("s3"); + + var sr1_GotHit = false; + var sr2_GotHit = false; + var sr3_GotHit = false; + + while (!sr1_GotHit || !sr2_GotHit || !sr3_GotHit) + { + sr1_GotHit = await sr1.GotStreamHit(); + sr2_GotHit = await sr2.GotStreamHit(); + sr3_GotHit = await sr3.GotStreamHit(); + } + + var sr1_host = await sr1.GetAddress(); + var sr2_host = await sr2.GetAddress(); + var sr3_host = await sr3.GetAddress(); + + Assert.Equal(Silo1, sr1_host); + Assert.Equal(Silo1, sr2_host); + Assert.Equal(Silo2, sr3_host); + + await Silo1Rebalancer.TriggerExchangeRequest(); + await Task.Delay(100); // leave some breathing room - may not be enough though, thats why this test is skippable + + var allowedDuration = TimeSpan.FromSeconds(3); + Stopwatch stopwatch = new(); + stopwatch.Start(); + + do + { + sr1_host = await sr1.GetAddress(); + sr2_host = await sr2.GetAddress(); + + Skip.If(stopwatch.Elapsed > allowedDuration); + } + while (sr1_host == Silo1 || sr2_host == Silo1); + + // refresh + sr1_host = await sr1.GetAddress(); + sr2_host = await sr2.GetAddress(); + sr3_host = await sr3.GetAddress(); + + Assert.Equal(Silo2, sr1_host); // SR1 is now in silo 2, as there is 1 pulling agent (which is moved to silo 2 by the streaming runtime) + Assert.Equal(Silo2, sr2_host); // SR2 is now in silo 2, as there is 1 pulling agent (which is moved to silo 2 by the streaming runtime) + Assert.Equal(Silo2, sr3_host); + + await ResetCounters(); + } + + public enum Scenario { _1, _2, _3, _4 } + + public interface IBase : IGrainWithStringKey + { + Task Ping(Scenario scenario); + Task GetAddress(); + } + public interface IA : IBase + { + Task FirstPing(Scenario scenario, SiloAddress silo1, SiloAddress silo2); + } + public interface IB : IBase { } + public interface IC : IBase + { + Task Ping(Scenario scenario, SiloAddress silo2); + } + public interface ICImmovable : IBase { } + public interface ID : IBase { } + public interface ISP : IGrainWithStringKey + { + Task FirstPing(); + Task StreamPing(); + Task GetAddress(); + } + public interface ISR : IGrainWithStringKey + { + Task Ping(); + Task GotStreamHit(); + Task GetAddress(); + } + + public abstract class GrainBase : Grain + { + public Task GetAddress() => Task.FromResult(GrainContext.Address.SiloAddress); + } + + public class A : GrainBase, IA + { + private SiloAddress _silo1; + private SiloAddress _silo2; + + public async Task FirstPing(Scenario scenario, SiloAddress silo1, SiloAddress silo2) + { + _silo1 = silo1; + _silo2 = silo2; + + switch (scenario) + { + case Scenario._1: + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, _silo2); + + await GrainFactory.GetGrain($"b{scenario}").Ping(scenario); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + } + break; + case Scenario._2: + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, _silo2); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + } + break; + case Scenario._3: + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, _silo2); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + } + break; + case Scenario._4: + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, _silo1); + await GrainFactory.GetGrain($"b{scenario}").Ping(scenario); + + RequestContext.Set(IPlacementDirector.PlacementHintKey, _silo2); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario, _silo2); + } + break; + default: throw new NotSupportedException(); + } + } + + public async Task Ping(Scenario scenario) + { + switch (scenario) + { + case Scenario._1: + { + await GrainFactory.GetGrain($"b{scenario}").Ping(scenario); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + } + break; + case Scenario._2: + { + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + } + break; + case Scenario._3: + { + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + } + break; + case Scenario._4: + { + await GrainFactory.GetGrain($"b{scenario}").Ping(scenario); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario); + await GrainFactory.GetGrain($"c{scenario}").Ping(scenario, _silo2); + } + break; + default: throw new NotSupportedException(); + } + } + } + + public class B : GrainBase, IB + { + public Task Ping(Scenario scenario) => + scenario switch + { + Scenario._1 => Task.CompletedTask, + Scenario._2 => GrainFactory.GetGrain($"c{scenario}").Ping(scenario), + Scenario._3 => GrainFactory.GetGrain($"c{scenario}").Ping(scenario), + Scenario._4 => Task.CompletedTask, + _ => throw new NotSupportedException(), + }; + } + + public class C : GrainBase, IC + { + public Task Ping(Scenario scenario) => + scenario switch + { + Scenario._1 => GrainFactory.GetGrain($"b{scenario}").Ping(scenario), + Scenario._2 => Task.CompletedTask, + Scenario._3 => Task.CompletedTask, + Scenario._4 => Task.CompletedTask, + _ => throw new NotSupportedException(), + }; + + public async Task Ping(Scenario scenario, SiloAddress silo2) + { + switch (scenario) + { + case Scenario._4: + { + RequestContext.Set(IPlacementDirector.PlacementHintKey, silo2); + await GrainFactory.GetGrain($"d{scenario}").Ping(scenario); + } + break; + default: throw new NotSupportedException(); + } + } + } + + [Immovable] + public class CImmovable : GrainBase, ICImmovable + { + public Task Ping(Scenario scenario) => + scenario switch + { + Scenario._3 => Task.CompletedTask, + _ => throw new NotSupportedException(), + }; + } + + public class D : GrainBase, ID + { + public Task Ping(Scenario scenario) => + scenario switch + { + Scenario._4 => Task.CompletedTask, + _ => throw new NotSupportedException(), + }; + } + + [Immovable] + public class SP : GrainBase, ISP + { + // We are just 'Immovable' on this type, because we just want it to push messages to the stream, + // as for some reason pushing to a stream via the cluster client isnt invoking the consumer grains. + + private IAsyncStream _stream; + + public override Task OnActivateAsync(CancellationToken cancellationToken) + { + _stream = this.GetStreamProvider(Fixture.StreamProviderName) + .GetStream(StreamId.Create(Fixture.StreamNamespaceName, this.GetPrimaryKeyString())); + + return Task.CompletedTask; + } + + public Task FirstPing() => GrainFactory.GetGrain(this.GetPrimaryKeyString()).Ping(); + public Task StreamPing() => _stream.OnNextAsync(Random.Shared.Next()); + } + + [ImplicitStreamSubscription(Fixture.StreamNamespaceName)] + public class SR : GrainBase, ISR + { + private bool _streamHit = false; + + public override async Task OnActivateAsync(CancellationToken cancellationToken) + { + var sp = this.GetStreamProvider(Fixture.StreamProviderName) + .GetStream(StreamId.Create(Fixture.StreamNamespaceName, this.GetPrimaryKeyString())); + + await sp.SubscribeAsync((_, _) => + { + _streamHit = true; + return Task.CompletedTask; + }); + } + + public Task Ping() => Task.CompletedTask; + public Task GotStreamHit() => Task.FromResult(_streamHit); + } + + public class Fixture : BaseTestClusterFixture + { + public const string StreamProviderName = "arsp"; + public const string StreamNamespaceName = "arns"; + + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.Options.InitialSilosCount = 2; + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) +#pragma warning disable ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + => hostBuilder + .Configure(o => + { + o.AssumeHomogenousSilosForTesting = true; + o.ClientGatewayShutdownNotificationTimeout = default; + }) + .Configure(o => + { + // Make these so that the timers practically never fire! We will invoke the protocol manually. + o.MinRebalancingDueTime = TimeSpan.FromSeconds(299); + o.MaxRebalancingDueTime = TimeSpan.FromSeconds(300); + // Make this practically zero, so we can invoke the protocol more than once without needing to put a delay in the tests. + o.RecoveryPeriod = TimeSpan.FromMilliseconds(1); + }) + .AddMemoryStreams(StreamProviderName, c => + { + c.ConfigurePartitioning(1); + c.ConfigureStreamPubSub(StreamPubSubType.ImplicitOnly); + }) + .AddActiveRebalancing() + .ConfigureServices(service => service.AddSingleton()); +#pragma warning restore ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + } + } +} \ No newline at end of file diff --git a/test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs b/test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs new file mode 100644 index 0000000000..8e9a59c508 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs @@ -0,0 +1,140 @@ +using System.Text; +using Orleans.Runtime.Placement.Rebalancing; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +public class FrequencyFilterTests +{ + [Fact] + public void GetExpectedTopK() + { + const int NumSamples = 10_000; + var sink = new UlongFrequentItemCollection(100); + var random = new Random(); + var distribution = new ZipfRejectionSampler(random, 1000, 0.5); + for (var i = 0; i < NumSamples; i++) + { + var sample = (ulong)distribution.Sample(); + sink.Add(new TestKey(sample)); + + if (i == 4 * NumSamples / 5) + { + sink.Remove(new TestKey(3)); + } + } + + var allCounters = sink.Elements.ToList(); + allCounters.Sort((left, right) => right.Count.CompareTo(left.Count)); + var sb = new StringBuilder(); + foreach (var (key, count, error) in allCounters) + { + if (error == 0) + { + sb.AppendLine($"{key.Key,3}: {count}"); + } + else + { + sb.AppendLine($"{key.Key,3}: {count} ε{error}"); + } + } + + var result = sb.ToString(); + Assert.NotEmpty(result); + } + + public readonly struct TestKey(ulong key) + { + private static ulong _nextKey; + public static TestKey GetNext() => new(_nextKey++); + public readonly ulong Key = key; + + public override string ToString() => $"[{Key}]"; + } + + private sealed class UlongFrequentItemCollection(int capacity) : FrequentItemCollection(capacity) + { + protected override ulong GetKey(in TestKey element) => element.Key; + public void Remove(in TestKey element) => RemoveCore(GetKey(element)); + } + + /// + /// Generates an approximate Zipf distribution. Previous method was 20x faster than MathNet.Numerics, but could only generate 250 samples/sec. + /// This approximate method can generate > 1,000,000 samples/sec. + /// + public class FastZipf + { + private static readonly Random SeededPrng = new(42); + + /// + /// Generate a zipf distribution. + /// + /// The random number generator to use. + /// The number of samples. + /// The skew. s=0 is a uniform distribution. As s increases, high-rank items become rapidly more likely than the rare low-ranked items. + /// N: the cardinality. The total number of items. + /// A zipf distribution. + public static long[] Generate(Random random, int sampleCount, double skew, int cardinality) + { + var sampler = new ZipfRejectionSampler(random, cardinality, skew); + + var samples = new long[sampleCount]; + for (var i = 0; i < sampleCount; i++) + { + samples[i] = sampler.Sample(); + } + + return samples; + } + + /// + /// Generate a zipf distribution. + /// + /// The number of samples. + /// The skew. s=0 is a uniform distribution. As s increases, high-rank items become rapidly more likely than the rare low-ranked items. + /// N: the cardinality. The total number of items. + /// A zipf distribution. + public static long[] Generate(int sampleCount, double skew, int cardinality) => Generate(SeededPrng, sampleCount, skew, cardinality); + } + + // https://jasoncrease.medium.com/rejection-sampling-the-zipf-distribution-6b359792cffa + public class ZipfRejectionSampler + { + private readonly Random _rand; + private readonly double _skew; + private readonly double _t; + + public ZipfRejectionSampler(Random random, long cardinality, double skew) + { + _rand = random; + _skew = skew; + _t = (Math.Pow(cardinality, 1 - skew) - skew) / (1 - skew); + } + + public long Sample() + { + while (true) + { + var invB = bInvCdf(_rand.NextDouble()); + var sampleX = (long)(invB + 1); + var yRand = _rand.NextDouble(); + var ratioTop = Math.Pow(sampleX, -_skew); + var ratioBottom = sampleX <= 1 ? 1 / _t : Math.Pow(invB, -_skew) / _t; + var rat = ratioTop / (ratioBottom * _t); + + if (yRand < rat) + { + return sampleX; + } + } + } + private double bInvCdf(double p) + { + return p * _t switch + { + <= 1 => p * _t, + _ => Math.Pow(p * _t * (1 - _skew) + _skew, 1 / (1 - _skew)) + }; + } + } +} \ No newline at end of file diff --git a/test/TesterInternal/ActiveRebalancingTests/FrequentEdgeCounterTests.cs b/test/TesterInternal/ActiveRebalancingTests/FrequentEdgeCounterTests.cs new file mode 100644 index 0000000000..8f550c2593 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/FrequentEdgeCounterTests.cs @@ -0,0 +1,99 @@ +using Orleans.Placement.Rebalancing; +using Orleans.Runtime; +using Orleans.Runtime.Placement.Rebalancing; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +[Alias("UnitTests.ActiveRebalancingTests.IMyGrain")] +public interface IMyActiveBalancingGrain : IGrainWithStringKey +{ + [Alias("GetValue")] + Task GetValue(); + + [Alias("GetValue1")] + Task GetValue(); +} + +[Alias("UnitTests.ActiveRebalancingTests.IMyGrain`1")] +public interface IMyActiveBalancingGrain : IGrainWithStringKey +{ + Task GetValue(); +} + +[TestCategory("Functional"), TestCategory("ActiveRebalancing")] +public class FrequentEdgeCounterTests +{ + private static readonly GrainId Id_A = GrainId.Create("A", Guid.NewGuid().ToString()); + private static readonly GrainId Id_B = GrainId.Create("B", Guid.NewGuid().ToString()); + private static readonly GrainId Id_C = GrainId.Create("C", Guid.NewGuid().ToString()); + private static readonly GrainId Id_D = GrainId.Create("D", Guid.NewGuid().ToString()); + private static readonly GrainId Id_E = GrainId.Create("E", Guid.NewGuid().ToString()); + private static readonly GrainId Id_F = GrainId.Create("F", Guid.NewGuid().ToString()); + + [Fact] + public void Add_ShouldIncrementCounter_WhenEdgeIsAdded() + { + var sink = new FrequentEdgeCounter(capacity: 10); + var edge = new Edge(new(Id_A, SiloAddress.Zero, true), new(Id_B, SiloAddress.Zero, true)); + + sink.Add(edge); + + var counters = sink.Elements.ToList(); + + Assert.Single(counters); + Assert.Equal(1u, counters[0].Count); + Assert.Equal(edge, counters[0].Element); + } + + [Fact] + public void Add_ShouldUpdateExistingCounter_WhenSameEdgeIsAddedAgain() + { + var sink = new FrequentEdgeCounter(capacity: 10); + var edge = new Edge(new(Id_A, SiloAddress.Zero, true), new(Id_B, SiloAddress.Zero, true)); + + sink.Add(edge); + sink.Add(edge); + + var counters = sink.Elements.ToList(); + + Assert.Single(counters); + Assert.Equal(2u, counters[0].Count); + Assert.Equal(edge, counters[0].Element); + } + + [Fact] + public void Add_ShouldRemoveMinCounter_WhenCapacityIsReached() + { + var sink = new FrequentEdgeCounter(capacity: 2); + + var edge1 = new Edge(new(Id_A, SiloAddress.Zero, true), new(Id_B, SiloAddress.Zero, true)); + var edge2 = new Edge(new(Id_C, SiloAddress.Zero, true), new(Id_D, SiloAddress.Zero, true)); + + sink.Add(edge1); + sink.Add(edge1); + sink.Add(edge2); + + Assert.Equal(2, sink.Count); + + var edge3 = new Edge(new(Id_E, SiloAddress.Zero, true), new(Id_F, SiloAddress.Zero, true)); + sink.Add(edge3); // should remove the minimum counter (edge2) since capacity is 2 + + var counters = sink.Elements.ToList(); + + Assert.Equal(2, counters.Count); + Assert.DoesNotContain(counters, c => c.Element == edge2); + } + + [Fact] + public void Remove_ShouldRemoveCounter_WhenEdgeIsRemoved() + { + var sink = new FrequentEdgeCounter(capacity: 10); + var edge = new Edge(new(Id_A, SiloAddress.Zero, true), new(Id_B, SiloAddress.Zero, true)); + + sink.Add(edge); + sink.Remove(edge); + + Assert.Empty(sink.Elements); + } +} diff --git a/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs b/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs new file mode 100644 index 0000000000..6ca3830421 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs @@ -0,0 +1,48 @@ +using Microsoft.Extensions.Options; +using Orleans.Runtime; +using Orleans.Configuration; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +[TestCategory("Functional"), TestCategory("ActiveRebalancing")] +public class OptionsTests +{ + [Fact] + public void ConstantsShouldNotChange() + { + Assert.Equal(10_000u, ActiveRebalancingOptions.DEFAULT_MAX_EDGE_COUNT); + Assert.Equal(TimeSpan.FromMinutes(1), ActiveRebalancingOptions.DEFAULT_MINUMUM_REBALANCING_DUE_TIME); + Assert.Equal(TimeSpan.FromMinutes(2), ActiveRebalancingOptions.DEFAULT_MAXIMUM_REBALANCING_DUE_TIME); + Assert.Equal(TimeSpan.FromMinutes(2), ActiveRebalancingOptions.DEFAULT_REBALANCING_PERIOD); + Assert.Equal(TimeSpan.FromMinutes(1), ActiveRebalancingOptions.DEFAULT_RECOVERY_PERIOD); + } + + [Theory] + [InlineData(0, 1, 1, 2, 1)] + [InlineData(1, 0, 1, 2, 1)] + [InlineData(1, 1, 0, 2, 1)] + [InlineData(1, 1, 1, 0, 1)] + [InlineData(1, 1, 1, 2, 0)] + [InlineData(1, 2, 1, 2, 1)] + [InlineData(1, 2, 1, 1, 2)] + public void InvalidOptionsShouldThrow( + uint topHeaviestCommunicationLinks, + int minimumRebalancingDueTimeMinutes, + int maximumRebalancingDueTimeMinutes, + int rebalancingPeriodMinutes, + int recoveryPeriodMinutes) + { + var options = new ActiveRebalancingOptions + { + MaxEdgeCount = topHeaviestCommunicationLinks, + MinRebalancingDueTime = TimeSpan.FromMinutes(minimumRebalancingDueTimeMinutes), + MaxRebalancingDueTime = TimeSpan.FromMinutes(maximumRebalancingDueTimeMinutes), + RebalancingPeriod = TimeSpan.FromMinutes(rebalancingPeriodMinutes), + RecoveryPeriod = TimeSpan.FromMinutes(recoveryPeriodMinutes) + }; + + var validator = new ActiveRebalancingOptionsValidator(Options.Create(options)); + Assert.Throws(validator.ValidateConfiguration); + } +} \ No newline at end of file diff --git a/test/TesterInternal/ActiveRebalancingTests/RebalancingTestBase.cs b/test/TesterInternal/ActiveRebalancingTests/RebalancingTestBase.cs new file mode 100644 index 0000000000..40a02ba973 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/RebalancingTestBase.cs @@ -0,0 +1,69 @@ +using Orleans.Placement.Rebalancing; +using Orleans.Runtime; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +public abstract class RebalancingTestBase : IAsyncLifetime where TFixture : BaseTestClusterFixture, new() +{ + private readonly TFixture _fixture; + + internal IInternalGrainFactory GrainFactory => _fixture.HostedCluster.InternalGrainFactory; + internal IActivationRebalancerSystemTarget Silo1Rebalancer { get; } + internal IActivationRebalancerSystemTarget Silo2Rebalancer { get; } + protected SiloAddress Silo1 { get; } + protected SiloAddress Silo2 { get; } + + public RebalancingTestBase(TFixture fixture) + { + _fixture = fixture; + + var silos = _fixture.HostedCluster.GetActiveSilos().Select(h => h.SiloAddress).OrderBy(s => s).ToArray(); + Silo1 = silos[0]; + Silo2 = silos[1]; + + Silo1Rebalancer = IActivationRebalancerSystemTarget.GetReference(GrainFactory, Silo1); + Silo2Rebalancer = IActivationRebalancerSystemTarget.GetReference(GrainFactory, Silo2); + } + + public virtual async Task InitializeAsync() + { + await GrainFactory.GetGrain(0).ForceActivationCollection(TimeSpan.FromSeconds(0)); + await ResetCounters(); + await AdjustActivationCountOffsets(); + } + + public virtual Task DisposeAsync() + { + return Task.CompletedTask; + } + + public async ValueTask ResetCounters() + { + await Silo1Rebalancer.ResetCounters(); + await Silo2Rebalancer.ResetCounters(); + } + + public async Task AdjustActivationCountOffsets() + { + // Account for imbalances in the initial activation counts. + Dictionary counts = []; + int max = 0; + foreach (var silo in (IEnumerable)_fixture.HostedCluster.Silos) + { + var sysTarget = GrainFactory.GetSystemTarget(Constants.ActivationRebalancerType, silo.SiloAddress); + var count = counts[silo.SiloAddress] = await sysTarget.GetActivationCount(); + max = Math.Max(max, count); + } + + foreach (var silo in (IEnumerable)_fixture.HostedCluster.Silos) + { + var sysTarget = GrainFactory.GetSystemTarget(Constants.ActivationRebalancerType, silo.SiloAddress); + var myCount = counts[silo.SiloAddress]; + await sysTarget.SetActivationCountOffset(max - myCount); + } + } + +} diff --git a/test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs b/test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs new file mode 100644 index 0000000000..d5a9070da0 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs @@ -0,0 +1,20 @@ +using Orleans.Runtime; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Rebalancing; + +namespace UnitTests.ActiveRebalancingTests; + +/// +/// Ignores client messages to make testing easier +/// +internal sealed class TestMessageFilter( + PlacementStrategyResolver strategyResolver, + IClusterManifestProvider clusterManifestProvider, + TimeProvider timeProvider) : IRebalancingMessageFilter +{ + private readonly RebalancingMessageFilter _messageFilter = new(strategyResolver, clusterManifestProvider, timeProvider); + + public bool IsAcceptable(Message message, out bool isSenderMigratable, out bool isTargetMigratable) => + _messageFilter.IsAcceptable(message, out isSenderMigratable, out isTargetMigratable) && + !message.SendingGrain.IsClient() && !message.TargetGrain.IsClient(); +} \ No newline at end of file diff --git a/test/TesterInternal/GrainDirectoryPartitionTests.cs b/test/TesterInternal/GrainDirectoryPartitionTests.cs index 00b9c06481..1e9f373295 100644 --- a/test/TesterInternal/GrainDirectoryPartitionTests.cs +++ b/test/TesterInternal/GrainDirectoryPartitionTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Orleans.Configuration; @@ -168,6 +169,8 @@ public SiloStatus GetApproximateSiloStatus(SiloAddress siloAddress) : new Dictionary(_content); } + public ImmutableArray GetActiveSilos() => _content.Keys.ToImmutableArray(); + public void SetSiloStatus(SiloAddress siloAddress, SiloStatus status) => _content[siloAddress] = status; public bool IsDeadSilo(SiloAddress silo) => GetApproximateSiloStatus(silo) == SiloStatus.Dead; diff --git a/test/TesterInternal/LivenessTests/ConsistentRingProviderTests.cs b/test/TesterInternal/LivenessTests/ConsistentRingProviderTests.cs index b998746aa4..d4c536251c 100644 --- a/test/TesterInternal/LivenessTests/ConsistentRingProviderTests.cs +++ b/test/TesterInternal/LivenessTests/ConsistentRingProviderTests.cs @@ -7,6 +7,7 @@ using Xunit.Abstractions; using TestExtensions; using System.Net; +using System.Collections.Immutable; namespace UnitTests.LivenessTests { @@ -181,6 +182,7 @@ public bool TryGetSiloName(SiloAddress siloAddress, out string siloName) } public bool UnSubscribeFromSiloStatusEvents(ISiloStatusListener observer) => _subscribers.Remove(observer); + public ImmutableArray GetActiveSilos() => [.. GetApproximateSiloStatuses(onlyActive: true).Keys]; } } } diff --git a/test/TesterInternal/TesterInternal.csproj b/test/TesterInternal/TesterInternal.csproj index f3d14e5802..db22ecfaa4 100644 --- a/test/TesterInternal/TesterInternal.csproj +++ b/test/TesterInternal/TesterInternal.csproj @@ -7,6 +7,7 @@ + From 915293c526974d8f8955e19ea1a2b3a0bc3a1c3a Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 20 May 2024 14:51:46 -0700 Subject: [PATCH 03/28] Try to improve candidate vertex max-heap perf --- .../Configuration/Options/MessagingOptions.cs | 2 +- .../IActivationRebalancerSystemTarget.cs | 11 + .../Options/SiloMessagingOptions.cs | 2 +- .../Rebalancing/ActivationRebalancer.cs | 214 +++++++++++------- .../Placement/Rebalancing/MaxHeap.cs | 109 +++++++-- .../CandidateVertexMaxHeapTests.cs | 47 ---- .../ActiveRebalancingTests/MaxHeapTests.cs | 76 +++++++ 7 files changed, 318 insertions(+), 143 deletions(-) delete mode 100644 test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs diff --git a/src/Orleans.Core/Configuration/Options/MessagingOptions.cs b/src/Orleans.Core/Configuration/Options/MessagingOptions.cs index 90663fb1a8..fdc47a9eac 100644 --- a/src/Orleans.Core/Configuration/Options/MessagingOptions.cs +++ b/src/Orleans.Core/Configuration/Options/MessagingOptions.cs @@ -11,7 +11,7 @@ public abstract class MessagingOptions /// /// The value. /// - private TimeSpan _responseTimeout = TimeSpan.FromSeconds(300); + private TimeSpan _responseTimeout = TimeSpan.FromSeconds(30); /// /// Gets or sets the default timeout before a request is assumed to have failed. diff --git a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs index 3727bc1ba2..1aee41822d 100644 --- a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs +++ b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs @@ -13,8 +13,10 @@ internal interface IActivationRebalancerSystemTarget : ISystemTarget static IActivationRebalancerSystemTarget GetReference(IGrainFactory grainFactory, SiloAddress targetSilo) => grainFactory.GetGrain(SystemTargetGrainId.Create(Constants.ActivationRebalancerType, targetSilo).GrainId); + [ResponseTimeout("00:10:00")] ValueTask TriggerExchangeRequest(); + [ResponseTimeout("00:10:00")] ValueTask AcceptExchangeRequest(AcceptExchangeRequest request); /// @@ -147,12 +149,21 @@ internal sealed class CandidateVertex [GenerateSerializer, Immutable] internal sealed class AcceptExchangeRequest(SiloAddress sendingSilo, ImmutableArray exchangeSet, int activationCountSnapshot) { + /// + /// The silo which is offering to transfer grains to us. + /// [Id(0)] public SiloAddress SendingSilo { get; } = sendingSilo; + /// + /// The set of grains which the sending silo is offering to transfer to us. + /// [Id(1)] public ImmutableArray ExchangeSet { get; } = exchangeSet; + /// + /// The activation count of the sending silo at the time of the exchange request. + /// [Id(2)] public int ActivationCountSnapshot { get; } = activationCountSnapshot; } diff --git a/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs b/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs index ca97e15884..7c51838835 100644 --- a/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/SiloMessagingOptions.cs @@ -12,7 +12,7 @@ public class SiloMessagingOptions : MessagingOptions /// /// . /// - private TimeSpan systemResponseTimeout = TimeSpan.FromSeconds(300); + private TimeSpan systemResponseTimeout = TimeSpan.FromSeconds(30); /// /// Gets or sets the number of parallel queues and attendant threads used by the silo to send outbound diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 5c0fd1fe2d..ff64828033 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -15,6 +15,9 @@ using Orleans.Internal; using Orleans.Configuration; using Orleans.Runtime.Utilities; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Serialization; +using System.Runtime.InteropServices; namespace Orleans.Runtime.Placement.Rebalancing; @@ -141,7 +144,10 @@ public async ValueTask TriggerExchangeRequest() var remoteRef = IActivationRebalancerSystemTarget.GetReference(_grainFactory, candidateSilo); var sw2 = ValueStopwatch.StartNew(); _logger.LogInformation("Sending AcceptExchangeRequest"); - var response = await remoteRef.AcceptExchangeRequest(new(Silo, exchangeSet, GetLocalActivationCount())); + AcceptExchangeRequest payload = new(Silo, exchangeSet, GetLocalActivationCount()); + var dummy = ActivationServices.GetRequiredService().SerializeToArray(payload); + _logger.LogInformation("Serializing AcceptExchangeRequest to {Size} bytes took {Elapsed}", dummy.Length, sw2.Elapsed); + var response = await remoteRef.AcceptExchangeRequest(payload); _logger.LogInformation("Sent AcceptExchangeRequest. It took {Elapsed}", sw2.Elapsed); switch (response.Type) @@ -186,6 +192,7 @@ public async ValueTask TriggerExchangeRequest() public async ValueTask AcceptExchangeRequest(AcceptExchangeRequest request) { + _logger.LogInformation("Received AcceptExchangeRequest from {Silo}", request.SendingSilo); if (request.SendingSilo.Equals(_currentExchangeSilo) && Silo.CompareTo(request.SendingSilo) <= 0) { // Reject the request, as we are already in the process of exchanging with the sending silo. @@ -210,16 +217,18 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha // This prevents other requests from interleaving. _currentExchangeSilo = request.SendingSilo; - var sw = ValueStopwatch.StartNew(); try { var remoteSet = request.ExchangeSet; + _logger.LogInformation("About to create candidate set"); var localSet = CreateCandidateSet(CreateLocalVertexEdges(), request.SendingSilo); + _logger.LogInformation("Created candidate set"); if (localSet.Count == 0) { // We have nothing to give back (very fringe case), so just accept the set. var set = remoteSet.Select(x => x.Id).ToImmutableArray(); + _logger.LogInformation("Finalizing protocol with empty local set"); await FinalizeProtocol(set, set, isReceiver: true); return new(AcceptExchangeResponse.ResponseType.Success, []); @@ -235,12 +244,21 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha var currentImbalance = 0; currentImbalance = CalculateImbalance(Direction.Unspecified); + _logger.LogInformation("Imbalance is {Imbalance}", currentImbalance); - var localHeap = new CandidateVertexMaxHeap(localSet); - var remoteHeap = new CandidateVertexMaxHeap(remoteSet); + var (localHeap, remoteHeap) = CreateCandidateHeaps(localSet, remoteSet); + _logger.LogInformation("Computing transfer set"); + var swTxs = ValueStopwatch.StartNew(); + var iterations = 0; while (true) { + if (++iterations % 128 == 0) + { + // Give other tasks a chance to execute periodically. + await Task.Delay(1); + } + if (localHeap.Count > 0 && remoteHeap.Count > 0) { var localVertex = localHeap.Peek(); @@ -287,9 +305,11 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha } } + _logger.LogInformation("2 Computing transfer set took {Elapsed}", swTxs.Elapsed); var unionSet = ImmutableArray.CreateBuilder(); var mySet = ImmutableArray.CreateBuilder(); var theirSet = ImmutableArray.CreateBuilder(); + swTxs.Restart(); foreach (var candidate in toMigrate) { if (candidate.TransferScore <= 0) @@ -311,7 +331,10 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha } } + _logger.LogInformation("Creating migration set took {Elapsed}", swTxs.Elapsed); + swTxs.Restart(); await FinalizeProtocol(mySet.ToImmutable(), unionSet.ToImmutable(), isReceiver: true); + _logger.LogInformation("Finalizing protocol based on provided set took {Elapsed}", swTxs.Elapsed); return new(AcceptExchangeResponse.ResponseType.Success, theirSet.ToImmutable()); @@ -328,10 +351,10 @@ bool TryMigrateLocalToRemote() return false; } - // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. var chosenVertex = localHeap.Pop(); if (chosenVertex.AccumulatedTransferScore <= 0) { + // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. return false; } @@ -341,47 +364,25 @@ bool TryMigrateLocalToRemote() remoteActivations++; currentImbalance = anticipatedImbalance; - var didUpdate = false; - foreach (var vertex in localHeap.UnorderedElements) + foreach (var (connectedVertex, transferScore) in chosenVertex.ConnectedVertices) { - var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); - if (connectedVertex == default) + switch (connectedVertex.Location) { - // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. - continue; + case VertexLocation.Local: + // Add the transfer score as these two vectors will now be remote to each other. + connectedVertex.AccumulatedTransferScore += transferScore; + localHeap.OnIncreaseElementPriority(connectedVertex); + break; + case VertexLocation.Remote: + // Subtract the transfer score as these two vectors will now be local to each other. + connectedVertex.AccumulatedTransferScore -= transferScore; + remoteHeap.OnDecreaseElementPriority(connectedVertex); + break; } - - // We add 'connectedVertex.TransferScore' to 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be remote to 'vertex' (because this is in the local heap). - vertex.AccumulatedTransferScore += connectedVertex.TransferScore; - didUpdate = true; - } - - if (didUpdate) - { - // Re-heapify the heap, since we changed priorities. - localHeap.Heapify(); } - didUpdate = false; - foreach (var vertex in remoteHeap.UnorderedElements) - { - var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); - if (connectedVertex == default) - { - // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. - continue; - } - - // We subtract 'connectedVertex.TransferScore' from 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be local to 'vertex' (because this is in the remote heap). - vertex.AccumulatedTransferScore -= connectedVertex.TransferScore; - didUpdate = true; - } - - if (didUpdate) - { - // Re-heapify the heap, since we changed priorities. - remoteHeap.Heapify(); - } + // We will perform any future operations assuming the vector is remote. + chosenVertex.Location = VertexLocation.Remote; return true; } @@ -400,8 +401,9 @@ bool TryMigrateRemoteToLocal() } var chosenVertex = remoteHeap.Pop(); - if (chosenVertex.AccumulatedTransferScore <= 0) // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. + if (chosenVertex.AccumulatedTransferScore <= 0) { + // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. return false; } @@ -410,48 +412,25 @@ bool TryMigrateRemoteToLocal() localActivations++; remoteActivations--; currentImbalance = anticipatedImbalance; - - var didUpdate = false; - foreach (var vertex in localHeap.UnorderedElements) + foreach (var (connectedVertex, transferScore) in chosenVertex.ConnectedVertices) { - var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); - if (connectedVertex == default) + switch (connectedVertex.Location) { - // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. - continue; + case VertexLocation.Local: + // Subtract the transfer score as these two vectors will now be local to each other. + connectedVertex.AccumulatedTransferScore -= transferScore; + localHeap.OnDecreaseElementPriority(connectedVertex); + break; + case VertexLocation.Remote: + // Add the transfer score as these two vectors will now be remote to each other. + connectedVertex.AccumulatedTransferScore += transferScore; + remoteHeap.OnIncreaseElementPriority(connectedVertex); + break; } - - // We subtract 'connectedVertex.TransferScore' from 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be local to 'vertex' (because this is in the local heap). - vertex.AccumulatedTransferScore -= connectedVertex.TransferScore; - didUpdate = true; } - if (didUpdate) - { - // Re-heapify the heap, since we changed priorities. - localHeap.Heapify(); - } - - didUpdate = false; - foreach (var vertex in remoteHeap.UnorderedElements) - { - var connectedVertex = chosenVertex.ConnectedVertices.FirstOrDefault(x => x.Id == vertex.Id); - if (connectedVertex == default) - { - // If no connection is present between [chosenVertex, vertex], we skip transfer score modification as the migration of 'chosenVertex', has not effect on this 'vertex'. - continue; - } - - // We add 'connectedVertex.TransferScore' to 'vertex.AccumulatedTransferScore', as the 'chosenVertex' will now be remote to 'vertex' (because this is in the remote heap) - vertex.AccumulatedTransferScore += connectedVertex.TransferScore; - didUpdate = true; - } - - if (didUpdate) - { - // Re-heapify the heap, since we changed priorities. - remoteHeap.Heapify(); - } + // We will perform any future operations assuming the vector is local. + chosenVertex.Location = VertexLocation.Local; return true; } @@ -468,11 +447,72 @@ int CalculateImbalance(Direction direction) return Math.Abs(Math.Abs(remoteActivations + rDelta) - Math.Abs(localActivations + lDelta)); } } + catch (Exception exception) + { + _logger.LogError(exception, "Error accepting exchange request."); + Debugger.Launch(); + throw; + } finally { - _logger.LogInformation("Computing transfer set took {Elapsed}.", sw.Elapsed); _currentExchangeSilo = null; } + + (MaxHeap LocalHeap, MaxHeap RemoteHeap) CreateCandidateHeaps(List localSet, ImmutableArray remoteSet) + { + Dictionary sourceIndex = []; + foreach (var element in localSet) + { + sourceIndex[element.Id] = element; + } + + foreach (var element in remoteSet) + { + sourceIndex[element.Id] = element; + } + + Dictionary index = []; + List localVertexList = []; + foreach (var element in localSet) + { + var vertex = CreateVertex(sourceIndex, index, element); + vertex.Location = VertexLocation.Local; + localVertexList.Add(vertex); + } + + List remoteVertexList = []; + foreach (var element in remoteSet) + { + var vertex = CreateVertex(sourceIndex, index, element); + vertex.Location = VertexLocation.Remote; + remoteVertexList.Add(vertex); + } + + var localHeap = new MaxHeap(localVertexList); + var remoteHeap = new MaxHeap(remoteVertexList); + return (localHeap, remoteHeap); + + static CandidateVertexHeapElement CreateVertex(Dictionary sourceIndex, Dictionary index, CandidateVertex element) + { + var vertex = GetOrAddVertex(index, element); + foreach (var connectedVertex in element.ConnectedVertices) + { + if (sourceIndex.TryGetValue(connectedVertex.Id, out var connected)) + { + vertex.ConnectedVertices.Add((GetOrAddVertex(index, connected), connectedVertex.TransferScore)); + } + } + + return vertex; + + static CandidateVertexHeapElement GetOrAddVertex(Dictionary index, CandidateVertex element) + { + ref var vertex = ref CollectionsMarshal.GetValueRefOrAddDefault(index, element.Id, out var exists); + vertex ??= new(element); + return vertex; + } + } + } } /// @@ -513,6 +553,7 @@ private async Task FinalizeProtocol(ImmutableArray idsToMigrate, Immuta LogErrorOnMigratingActivations(aggEx, Silo); } + _logger.LogInformation("Waiting for {Count} grains to migrate took {Elapsed}", affectedIds.Length, sw1.Elapsed); if (isReceiver) { // Stamp this silos exchange for a potential next pair exchange request. @@ -521,12 +562,19 @@ private async Task FinalizeProtocol(ImmutableArray idsToMigrate, Immuta } var sw = ValueStopwatch.StartNew(); + var iterations = 0; if (affectedIds.Length != 0) { // Avoid mutating the source while enumerating it. var toRemove = new List(); foreach (var (edge, count, error) in _edgeWeights.Elements) { + if (++iterations % 128 == 0) + { + // Give other tasks a chance to execute periodically. + await Task.Delay(1); + } + if (edge.ContainsAny(affectedIds)) { toRemove.Add(edge); @@ -535,6 +583,12 @@ private async Task FinalizeProtocol(ImmutableArray idsToMigrate, Immuta foreach (var edge in toRemove) { + if (++iterations % 128 == 0) + { + // Give other tasks a chance to execute periodically. + await Task.Delay(1); + } + // Totally remove this counter, as one or both vertices has migrated. By not doing this it would skew results for the next protocol cycle. // We remove only the affected counters, as there could be other counters that 'this' silo has connections with another silo (which is not part of this exchange cycle). _edgeWeights.Remove(edge); diff --git a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs index a0ee133e7d..885ac9ed16 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs @@ -8,9 +8,28 @@ namespace Orleans.Runtime.Placement.Rebalancing; -internal sealed class CandidateVertexMaxHeap(ICollection values) : MaxHeap(values) +internal enum VertexLocation { - protected override int Compare(CandidateVertex left, CandidateVertex right) => -left.AccumulatedTransferScore.CompareTo(right.AccumulatedTransferScore); + Local, + Remote +} + +internal sealed class CandidateVertexHeapElement(CandidateVertex value) : IHeapElement +{ + public CandidateVertex Vertex { get; } = value; + public List<(CandidateVertexHeapElement Element, long TransferScore)> ConnectedVertices { get; } = []; + public GrainId Id => Vertex.Id; + public long AccumulatedTransferScore { get => Vertex.AccumulatedTransferScore; set => Vertex.AccumulatedTransferScore = value; } + public VertexLocation Location { get; set; } + int IHeapElement.HeapIndex { get; set; } + int IHeapElement.CompareTo(CandidateVertexHeapElement other) + => Vertex.AccumulatedTransferScore.CompareTo(other.Vertex.AccumulatedTransferScore); +} + +internal interface IHeapElement where TElement : notnull +{ + int HeapIndex { get; set; } + int CompareTo(TElement other); } /// @@ -22,7 +41,7 @@ internal sealed class CandidateVertexMaxHeap(ICollection values /// Elements with the lowest priority get removed first. /// [DebuggerDisplay("Count = {Count}")] -internal abstract class MaxHeap where TElement : notnull +internal sealed class MaxHeap where TElement : notnull, IHeapElement { /// /// Represents an implicit heap-ordered complete d-ary tree, stored as an array. @@ -45,8 +64,6 @@ internal abstract class MaxHeap where TElement : notnull /// private const int Log2Arity = 2; - protected abstract int Compare(TElement left, TElement right); - #if DEBUG static MaxHeap() { @@ -73,12 +90,20 @@ public MaxHeap(ICollection items) _size = items.Count; var nodes = new TElement[_size]; items.CopyTo(nodes, 0); - _nodes = nodes; + for (var i = 0; i< nodes.Length; i++) + { + nodes[i].HeapIndex = i; + } + _nodes = nodes; if (_size > 1) { Heapify(); } + else if (_size == 1) + { + _nodes[0]!.HeapIndex = 0; + } } /// @@ -115,6 +140,7 @@ public TElement Pop() var element = _nodes[0]!; RemoveRootNode(); + element.HeapIndex = -1; return element; void RemoveRootNode() @@ -144,6 +170,30 @@ void RemoveRootNode() /// private static int GetFirstChildIndex(int index) => (index << Log2Arity) + 1; + public void OnDecreaseElementPriority(TElement element) + { + // If the element has already been removed from the heap, this is a no-op. + if (element.HeapIndex < 0) + { + return; + } + + // The element's priority has decreased, so move it down as necessary to restore the heap property. + MoveDown(element, element.HeapIndex); + } + + public void OnIncreaseElementPriority(TElement element) + { + // If the element has already been removed from the heap, this is a no-op. + if (element.HeapIndex <= 0) + { + return; + } + + // The element's priority has increased, so move it down as necessary to restore the heap property. + MoveUp(element, element.HeapIndex); + } + /// /// Converts an unordered list into a heap. /// @@ -167,6 +217,35 @@ public void Heapify() /// public UnorderedElementEnumerable UnorderedElements => new(this); + /// + /// Moves a node up in the tree to restore heap order. + /// + private void MoveUp(TElement node, int nodeIndex) + { + Debug.Assert(0 <= nodeIndex && nodeIndex < _size); + + var nodes = _nodes; + + while (nodeIndex > 0) + { + var parentIndex = GetParentIndex(nodeIndex); + var parentNode = nodes[parentIndex]!; + + if (node.CompareTo(parentNode) <= 0) + { + // The parent is more larger than the current node. + break; + } + + nodes[nodeIndex] = parentNode; + parentNode.HeapIndex = nodeIndex; + nodeIndex = parentIndex; + } + + nodes[nodeIndex] = node; + node.HeapIndex = nodeIndex; + } + /// /// Moves a node down in the tree to restore heap order. /// @@ -185,33 +264,35 @@ private void MoveDown(TElement node, int nodeIndex) while ((i = GetFirstChildIndex(nodeIndex)) < size) { // Find the child node with the maximal priority - var minChild = nodes[i]!; - var minChildIndex = i; + var maxChild = nodes[i]!; + var maxChildIndex = i; var childIndexUpperBound = Math.Min(i + Arity, size); while (++i < childIndexUpperBound) { var nextChild = nodes[i]!; - if (Compare(nextChild, minChild) < 0) + if (nextChild.CompareTo(maxChild) > 0) { - minChild = nextChild; - minChildIndex = i; + maxChild = nextChild; + maxChildIndex = i; } } // Heap property is satisfied; insert node in this location. - if (Compare(node, minChild) <= 0) + if (node.CompareTo(maxChild) >= 0) { break; } // Move the maximal child up by one node and // continue recursively from its location. - nodes[nodeIndex] = minChild; - nodeIndex = minChildIndex; + nodes[nodeIndex] = maxChild; + maxChild.HeapIndex = nodeIndex; + nodeIndex = maxChildIndex; } nodes[nodeIndex] = node; + node.HeapIndex = nodeIndex; } /// diff --git a/test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs b/test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs deleted file mode 100644 index bf35feeb31..0000000000 --- a/test/TesterInternal/ActiveRebalancingTests/CandidateVertexMaxHeapTests.cs +++ /dev/null @@ -1,47 +0,0 @@ -using Orleans.Placement.Rebalancing; -using Orleans.Runtime.Placement.Rebalancing; -using Xunit; - -namespace UnitTests.ActiveRebalancingTests; - -public sealed class CandidateVertexMaxHeapTests -{ - [Fact] - void HeapPropertyIsMaintained() - { - var edges = new CandidateVertex[100]; - for (int i = 0; i < edges.Length; i++) - { - edges[i] = new CandidateVertex { AccumulatedTransferScore = i }; - } - - Random.Shared.Shuffle(edges); - var heap = new CandidateVertexMaxHeap(edges); - Assert.Equal(100, heap.Count); - Assert.Equal(99, heap.Peek().AccumulatedTransferScore); - Assert.Equal(99, heap.Peek().AccumulatedTransferScore); - Assert.Equal(99, heap.Pop().AccumulatedTransferScore); - Assert.Equal(98, heap.Pop().AccumulatedTransferScore); - Assert.Equal(98, heap.Count); - Assert.Equal(98, heap.UnorderedElements.Count()); - - // Randomly re-assign priorities to edges - var newScore = 1000; - var elements = heap.UnorderedElements.ToArray(); - Random.Shared.Shuffle(edges); - - foreach (var element in elements) - { - element.AccumulatedTransferScore = newScore--; - } - - heap.Heapify(); - - Assert.Equal(1000, heap.Peek().AccumulatedTransferScore); - Assert.Equal(1000, heap.Peek().AccumulatedTransferScore); - Assert.Equal(1000, heap.Pop().AccumulatedTransferScore); - Assert.Equal(999, heap.Pop().AccumulatedTransferScore); - Assert.Equal(96, heap.Count); - Assert.Equal(96, heap.UnorderedElements.Count()); - } -} diff --git a/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs b/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs new file mode 100644 index 0000000000..1f3600e02b --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs @@ -0,0 +1,76 @@ +using Orleans.Runtime.Placement.Rebalancing; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +public sealed class MaxHeapTests +{ + public class MyHeapElement(int value) : IHeapElement + { + public int Value { get; set; } = value; + + public int HeapIndex { get; set; } + + public int CompareTo(MyHeapElement other) => Value.CompareTo(other.Value); + public override string ToString() => $"{Value} @ {HeapIndex}"; + } + + [Fact] + public void HeapPropertyIsMaintained() + { + var edges = new MyHeapElement[100]; + for (int i = 0; i < edges.Length; i++) + { + edges[i] = new MyHeapElement(i); + } + + Random.Shared.Shuffle(edges); + var heap = new MaxHeap(edges); + Assert.Equal(100, heap.Count); + Assert.Equal(99, heap.Peek().Value); + Assert.Equal(99, heap.Peek().Value); + Assert.Equal(99, heap.Pop().Value); + Assert.Equal(98, heap.Pop().Value); + Assert.Equal(98, heap.Count); + Assert.Equal(98, heap.UnorderedElements.Count()); + + var unorderedElements = heap.UnorderedElements.ToArray(); + var edge = unorderedElements[Random.Shared.Next(unorderedElements.Length)]; + edge.Value = 2000; + heap.OnIncreaseElementPriority(edge); + Assert.Equal(2000, heap.Peek().Value); + + // Randomly re-assign priorities to edges + var newScore = 100; + var elements = heap.UnorderedElements.ToArray(); + Random.Shared.Shuffle(elements); + foreach (var element in elements) + { + var originalValue = element.Value; + element.Value = newScore--; + if (element.Value > originalValue) + { + heap.OnIncreaseElementPriority(element); + } + else + { + heap.OnDecreaseElementPriority(element); + } + } + + Assert.Equal(98, heap.UnorderedElements.Count()); + var allElements = new List(); + while (heap.Count > 0) + { + allElements.Add(heap.Pop()); + } + + Assert.Equal(98, allElements.Count); + + var copy = allElements.ToList(); + copy.Sort((a, b) => b.Value.CompareTo(a.Value)); + var expected = string.Join(", ", Enumerable.Range(0, 98).Select(i => 100 - i)); + var actual = string.Join(", ", allElements.Select(c => c.Value)); + Assert.Equal(expected, actual); + } +} From d40485bf25a8097e5e5ca795fa22dc31dee73aff Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 20 May 2024 19:07:38 -0700 Subject: [PATCH 04/28] WIP --- .../IActivationRebalancerSystemTarget.cs | 27 +- .../Rebalancing/ActivationRebalancer.cs | 300 ++++++++---------- .../Placement/Rebalancing/MaxHeap.cs | 20 +- 3 files changed, 153 insertions(+), 194 deletions(-) diff --git a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs index 1aee41822d..a958713a1e 100644 --- a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs +++ b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs @@ -60,22 +60,6 @@ static IActivationRebalancerSystemTarget GetReference(IGrainFactory grainFactory /// Returns a copy of this but with flipped sources and targets. /// public Edge Flip() => new(source: Target, target: Source); - - /// - /// Checks if any of the is part of this counter. - /// - public readonly bool ContainsAny(ImmutableArray grainIds) - { - foreach (var grainId in grainIds) - { - if (Source.Id == grainId || Target.Id == grainId) - { - return true; - } - } - - return false; - } } /// @@ -169,16 +153,19 @@ internal sealed class AcceptExchangeRequest(SiloAddress sendingSilo, ImmutableAr } [GenerateSerializer, Immutable] -internal sealed class AcceptExchangeResponse(AcceptExchangeResponse.ResponseType type, ImmutableArray exchangeSet) +internal sealed class AcceptExchangeResponse(AcceptExchangeResponse.ResponseType type, ImmutableArray acceptedGrains, ImmutableArray givenGrains) { - public static readonly AcceptExchangeResponse CachedExchangedRecently = new(ResponseType.ExchangedRecently, []); - public static readonly AcceptExchangeResponse CachedMutualExchangeAttempt = new(ResponseType.MutualExchangeAttempt, []); + public static readonly AcceptExchangeResponse CachedExchangedRecently = new(ResponseType.ExchangedRecently, [], []); + public static readonly AcceptExchangeResponse CachedMutualExchangeAttempt = new(ResponseType.MutualExchangeAttempt, [], []); [Id(0)] public ResponseType Type { get; } = type; [Id(1)] - public ImmutableArray ExchangeSet { get; } = exchangeSet; + public ImmutableArray AcceptedGrainIds { get; } = acceptedGrains; + + [Id(2)] + public ImmutableArray GivenGrainIds { get; } = givenGrains; [GenerateSerializer] public enum ResponseType diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index ff64828033..55fe68617b 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -15,9 +15,8 @@ using Orleans.Internal; using Orleans.Configuration; using Orleans.Runtime.Utilities; -using Microsoft.Extensions.DependencyInjection; -using Orleans.Serialization; using System.Runtime.InteropServices; +using System.Runtime.ExceptionServices; namespace Orleans.Runtime.Placement.Rebalancing; @@ -102,6 +101,12 @@ private void RegisterOrUpdateTimer(TimeSpan dueTime) public async ValueTask TriggerExchangeRequest() { + if (_currentExchangeSilo is not null) + { + // Skip this round if we are already in the process of exchanging with another silo. + return; + } + var silos = _siloStatusOracle.GetActiveSilos(); if (silos.Length == 1) { @@ -126,8 +131,8 @@ public async ValueTask TriggerExchangeRequest() return; } - (var candidateSilo, var exchangeSet, var _) = set; - if (exchangeSet.Length == 0) + (var candidateSilo, var offeredGrains, var _) = set; + if (offeredGrains.Count == 0) { countWithNoExchangeSet++; LogExchangeSetIsEmpty(candidateSilo); @@ -144,17 +149,14 @@ public async ValueTask TriggerExchangeRequest() var remoteRef = IActivationRebalancerSystemTarget.GetReference(_grainFactory, candidateSilo); var sw2 = ValueStopwatch.StartNew(); _logger.LogInformation("Sending AcceptExchangeRequest"); - AcceptExchangeRequest payload = new(Silo, exchangeSet, GetLocalActivationCount()); - var dummy = ActivationServices.GetRequiredService().SerializeToArray(payload); - _logger.LogInformation("Serializing AcceptExchangeRequest to {Size} bytes took {Elapsed}", dummy.Length, sw2.Elapsed); - var response = await remoteRef.AcceptExchangeRequest(payload); + var response = await remoteRef.AcceptExchangeRequest(new (Silo, offeredGrains.ToImmutableArray(), GetLocalActivationCount())); _logger.LogInformation("Sent AcceptExchangeRequest. It took {Elapsed}", sw2.Elapsed); switch (response.Type) { case AcceptExchangeResponse.ResponseType.Success: // Exchange was successful, no need to iterate over another candidate. - await FinalizeProtocol(response.ExchangeSet, exchangeSet.Select(x => x.Id).Union(response.ExchangeSet).ToImmutableArray(), isReceiver: false); + await FinalizeProtocol(response.AcceptedGrainIds, response.GivenGrainIds, isReceiver: false, candidateSilo); return; case AcceptExchangeResponse.ResponseType.ExchangedRecently: // The remote silo has been recently involved in another exchange, try the next best candidate. @@ -190,8 +192,31 @@ public async ValueTask TriggerExchangeRequest() private int GetLocalActivationCount() => _activationDirectory.Count + _activationCountOffset; + private struct AttachDebuggerOnFirstChance : IDisposable + { + public AttachDebuggerOnFirstChance() + { + AppDomain.CurrentDomain.FirstChanceException += CurrentDomain_FirstChanceException; + } + + public readonly void Dispose() + { + AppDomain.CurrentDomain.FirstChanceException -= CurrentDomain_FirstChanceException; + } + + private void CurrentDomain_FirstChanceException(object? sender, FirstChanceExceptionEventArgs e) + { + if (e.Exception is IndexOutOfRangeException) + { + Debugger.Launch(); + } + } + } + public async ValueTask AcceptExchangeRequest(AcceptExchangeRequest request) { + using var _ = new AttachDebuggerOnFirstChance(); + _logger.LogInformation("Received AcceptExchangeRequest from {Silo}", request.SendingSilo); if (request.SendingSilo.Equals(_currentExchangeSilo) && Silo.CompareTo(request.SendingSilo) <= 0) { @@ -207,7 +232,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha } var lastExchangeElapsed = _lastExchangedStopwatch.Elapsed; - if (lastExchangeElapsed < _options.RecoveryPeriod) + if (lastExchangeElapsed < _options.RecoveryPeriod || _currentExchangeSilo != null) { LogExchangedRecently(request.SendingSilo, lastExchangeElapsed, _options.RecoveryPeriod); return AcceptExchangeResponse.CachedExchangedRecently; @@ -219,32 +244,21 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha try { + var acceptedGrains = ImmutableArray.CreateBuilder(); + var givingGrains = ImmutableArray.CreateBuilder(); var remoteSet = request.ExchangeSet; _logger.LogInformation("About to create candidate set"); var localSet = CreateCandidateSet(CreateLocalVertexEdges(), request.SendingSilo); _logger.LogInformation("Created candidate set"); - if (localSet.Count == 0) - { - // We have nothing to give back (very fringe case), so just accept the set. - var set = remoteSet.Select(x => x.Id).ToImmutableArray(); - _logger.LogInformation("Finalizing protocol with empty local set"); - await FinalizeProtocol(set, set, isReceiver: true); - - return new(AcceptExchangeResponse.ResponseType.Success, []); - } - - List<(GrainId Grain, SiloAddress Silo, long TransferScore)> toMigrate = []; - // We need to determine 2 subsets: // - One that originates from sending silo (request.ExchangeSet) and will be (partially) accepted from this silo. // - One that originates from this silo (candidateSet) and will be (fully) accepted from the sending silo. var remoteActivations = request.ActivationCountSnapshot; var localActivations = GetLocalActivationCount(); - var currentImbalance = 0; - currentImbalance = CalculateImbalance(Direction.Unspecified); - _logger.LogInformation("Imbalance is {Imbalance}", currentImbalance); + var imbalance = CalculateImbalance(remoteActivations, localActivations); + _logger.LogInformation("Imbalance is {Imbalance} (remote: {RemoteCount} vs local {LocalCount})", imbalance, remoteActivations, localActivations); var (localHeap, remoteHeap) = CreateCandidateHeaps(localSet, remoteSet); @@ -259,84 +273,32 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha await Task.Delay(1); } - if (localHeap.Count > 0 && remoteHeap.Count > 0) - { - var localVertex = localHeap.Peek(); - var remoteVertex = remoteHeap.Peek(); - - if (localVertex.AccumulatedTransferScore > remoteVertex.AccumulatedTransferScore) - { - if (TryMigrateLocalToRemote()) continue; - if (TryMigrateRemoteToLocal()) continue; - } - else if (localVertex.AccumulatedTransferScore < remoteVertex.AccumulatedTransferScore) - { - if (TryMigrateRemoteToLocal()) continue; - if (TryMigrateLocalToRemote()) continue; - } - else - { - // Other than testing scenarios with a handful of activations, it should be rare that this happens. If the transfer scores are equal, than we check the anticipated imbalances - // for both cases, and proceed with whichever lowers the overall imbalance, even though the other option could still be within the tolerance margin. - // The imbalance check is the first step micro-optimization, which doesn't necessarily mean that the migration direction (L2R, R2L) will happen, that is still - // determined within the migration methods. In case both anticipated imbalances are also equal, we have to pick one, and we stick for consistency with L2R in that case. - var l2r_anticipatedImbalance = CalculateImbalance(Direction.LocalToRemote); - var r2l_anticipatedImbalance = CalculateImbalance(Direction.RemoteToLocal); - - if (l2r_anticipatedImbalance <= r2l_anticipatedImbalance) - { - if (TryMigrateLocalToRemote()) continue; - if (TryMigrateRemoteToLocal()) continue; - } - else - { - if (TryMigrateRemoteToLocal()) continue; - if (TryMigrateLocalToRemote()) continue; - } - } - } - else + // If more is gained by giving grains to the remote silo than taking from it, we will try giving first. + var localScore = localHeap.FirstOrDefault()?.AccumulatedTransferScore ?? 0; + var remoteScore = remoteHeap.FirstOrDefault()?.AccumulatedTransferScore ?? 0; + if (localScore > remoteScore || localActivations > remoteActivations) { if (TryMigrateLocalToRemote()) continue; if (TryMigrateRemoteToLocal()) continue; - - // Both heaps are empty, at this point we are done. - break; } - } - - _logger.LogInformation("2 Computing transfer set took {Elapsed}", swTxs.Elapsed); - var unionSet = ImmutableArray.CreateBuilder(); - var mySet = ImmutableArray.CreateBuilder(); - var theirSet = ImmutableArray.CreateBuilder(); - swTxs.Restart(); - foreach (var candidate in toMigrate) - { - if (candidate.TransferScore <= 0) + else { - continue; + if (TryMigrateRemoteToLocal()) continue; + if (TryMigrateLocalToRemote()) continue; } - if (candidate.Silo.Equals(Silo)) - { - // Add to the subset that should migrate to 'this' silo. - mySet.Add(candidate.Grain); - unionSet.Add(candidate.Grain); - } - else if (candidate.Silo.Equals(request.SendingSilo)) - { - // Add to the subset to send back to 'remote' silo (the actual migration will be handled there) - theirSet.Add(candidate.Grain); - unionSet.Add(candidate.Grain); - } + // No more migrations can be made, so the candidate set has been calculated. + break; } - _logger.LogInformation("Creating migration set took {Elapsed}", swTxs.Elapsed); + _logger.LogInformation("Computing transfer set took {Elapsed}. Anticipated imbalance after transfer is {AnticipatedImbalance}", swTxs.Elapsed, imbalance); swTxs.Restart(); - await FinalizeProtocol(mySet.ToImmutable(), unionSet.ToImmutable(), isReceiver: true); + var giving = givingGrains.ToImmutable(); + var accepting = acceptedGrains.ToImmutable(); + await FinalizeProtocol(giving, accepting, isReceiver: true, request.SendingSilo); _logger.LogInformation("Finalizing protocol based on provided set took {Elapsed}", swTxs.Elapsed); - return new(AcceptExchangeResponse.ResponseType.Success, theirSet.ToImmutable()); + return new(AcceptExchangeResponse.ResponseType.Success, accepting, giving); bool TryMigrateLocalToRemote() { @@ -345,8 +307,8 @@ bool TryMigrateLocalToRemote() return false; } - var anticipatedImbalance = CalculateImbalance(Direction.LocalToRemote); - if (anticipatedImbalance > currentImbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + var anticipatedImbalance = CalculateImbalance(localActivations - 1, remoteActivations + 1); + if (anticipatedImbalance > imbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) { return false; } @@ -358,11 +320,11 @@ bool TryMigrateLocalToRemote() return false; } - toMigrate.Add(new(chosenVertex.Id, request.SendingSilo, chosenVertex.AccumulatedTransferScore)); + givingGrains.Add(chosenVertex.Id); localActivations--; remoteActivations++; - currentImbalance = anticipatedImbalance; + imbalance = anticipatedImbalance; foreach (var (connectedVertex, transferScore) in chosenVertex.ConnectedVertices) { @@ -394,8 +356,8 @@ bool TryMigrateRemoteToLocal() return false; } - var anticipatedImbalance = CalculateImbalance(Direction.RemoteToLocal); - if (anticipatedImbalance > currentImbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + var anticipatedImbalance = CalculateImbalance(localActivations + 1, remoteActivations - 1); + if (anticipatedImbalance > imbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) { return false; } @@ -407,11 +369,11 @@ bool TryMigrateRemoteToLocal() return false; } - toMigrate.Add(new(chosenVertex.Id, Silo, chosenVertex.AccumulatedTransferScore)); + acceptedGrains.Add(chosenVertex.Id); localActivations++; remoteActivations--; - currentImbalance = anticipatedImbalance; + imbalance = anticipatedImbalance; foreach (var (connectedVertex, transferScore) in chosenVertex.ConnectedVertices) { switch (connectedVertex.Location) @@ -435,22 +397,11 @@ bool TryMigrateRemoteToLocal() return true; } - int CalculateImbalance(Direction direction) - { - (var rDelta, var lDelta) = direction switch - { - Direction.LocalToRemote => (1, -1), - Direction.RemoteToLocal => (-1, 1), - _ => (0, 0) - }; - - return Math.Abs(Math.Abs(remoteActivations + rDelta) - Math.Abs(localActivations + lDelta)); - } + static int CalculateImbalance(int left, int right) => Math.Abs(Math.Abs(left) - Math.Abs(right)); } catch (Exception exception) { _logger.LogError(exception, "Error accepting exchange request."); - Debugger.Launch(); throw; } finally @@ -517,91 +468,98 @@ static CandidateVertexHeapElement GetOrAddVertex(Dictionary /// - /// Initiates the actual migration process of to 'this' silo. + /// Initiates the actual migration process of to 'this' silo. /// If it proceeds to update . - /// Updates the affected counters within to reflect all . + /// Updates the affected counters within to reflect all . /// /// - /// The grain ids to migrate. - /// All grains ids that were affected from both sides. + /// The grain ids to migrate to the remote host. + /// The grain ids to which are migrating to the local host. /// Is the caller, the protocol receiver or not. - private async Task FinalizeProtocol(ImmutableArray idsToMigrate, ImmutableArray affectedIds, bool isReceiver) + private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArray accepting, bool isReceiver, SiloAddress targetSilo) { - if (idsToMigrate.Length > 0) + if (giving.Length == 0) { - // The protocol concluded that 'this' silo should take on 'set', so we hint to the director accordingly. - RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo); - List migrationTasks = []; + LogProtocolFinalized(giving.Length); + return; + } - var sw1 = ValueStopwatch.StartNew(); - _logger.LogInformation("Telling {Count} grains to migrate", affectedIds.Length); - foreach (var grainId in idsToMigrate) - { - migrationTasks.Add(_grainFactory.GetGrain(grainId).Cast().MigrateOnIdle().AsTask()); - } - _logger.LogInformation("Telling {Count} grains to migrate took {Elapsed}", affectedIds.Length, sw1.Elapsed); + // The protocol concluded that 'this' silo should take on 'set', so we hint to the director accordingly. + RequestContext.Set(IPlacementDirector.PlacementHintKey, targetSilo); + List migrationTasks = []; - try - { - await Task.WhenAll(migrationTasks); - } - catch (Exception) - { - // This should happen rarely, but at this point we cant really do much, as its out of our control. - // Even if some fail, at the end the algorithm will run again and eventually succeed with moving all activations were they belong. - var aggEx = new AggregateException(migrationTasks.Select(t => t.Exception).Where(ex => ex is not null)!); - LogErrorOnMigratingActivations(aggEx, Silo); - } + var sw1 = ValueStopwatch.StartNew(); + _logger.LogInformation("Telling {Count} grains to migrate from {LocalSilo} to {TargetSilo}", giving.Length, Silo, targetSilo); + foreach (var grainId in giving) + { + migrationTasks.Add(_grainFactory.GetGrain(grainId).Cast().MigrateOnIdle().AsTask()); + } + _logger.LogInformation("Telling {Count} grains to migrate took {Elapsed}", giving.Length, sw1.Elapsed); - _logger.LogInformation("Waiting for {Count} grains to migrate took {Elapsed}", affectedIds.Length, sw1.Elapsed); - if (isReceiver) - { - // Stamp this silos exchange for a potential next pair exchange request. - _lastExchangedStopwatch.Restart(); - } + try + { + await Task.WhenAll(migrationTasks); + } + catch (Exception) + { + // This should happen rarely, but at this point we cant really do much, as its out of our control. + // Even if some fail, at the end the algorithm will run again and eventually succeed with moving all activations were they belong. + var aggEx = new AggregateException(migrationTasks.Select(t => t.Exception).Where(ex => ex is not null)!); + LogErrorOnMigratingActivations(aggEx, Silo); } + _logger.LogInformation("Waiting for {Count} grains to migrate took {Elapsed}", giving.Length, sw1.Elapsed); + if (isReceiver) + { + // Stamp this silos exchange for a potential next pair exchange request. + _lastExchangedStopwatch.Restart(); + } + + // Avoid mutating the source while enumerating it. var sw = ValueStopwatch.StartNew(); var iterations = 0; - if (affectedIds.Length != 0) + var toRemove = new List(); + + var affected = new HashSet(giving.Length + accepting.Length); + foreach (var id in accepting) { - // Avoid mutating the source while enumerating it. - var toRemove = new List(); - foreach (var (edge, count, error) in _edgeWeights.Elements) - { - if (++iterations % 128 == 0) - { - // Give other tasks a chance to execute periodically. - await Task.Delay(1); - } + affected.Add(id); + } - if (edge.ContainsAny(affectedIds)) - { - toRemove.Add(edge); - } - } + foreach (var id in giving) + { + affected.Add(id); + } - foreach (var edge in toRemove) + foreach (var (edge, count, error) in _edgeWeights.Elements) + { + if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id)) { - if (++iterations % 128 == 0) - { - // Give other tasks a chance to execute periodically. - await Task.Delay(1); - } + toRemove.Add(edge); + } + } - // Totally remove this counter, as one or both vertices has migrated. By not doing this it would skew results for the next protocol cycle. - // We remove only the affected counters, as there could be other counters that 'this' silo has connections with another silo (which is not part of this exchange cycle). - _edgeWeights.Remove(edge); + foreach (var edge in toRemove) + { + if (++iterations % 128 == 0) + { + // Give other tasks a chance to execute periodically. + await Task.Delay(1); } + + // Totally remove this counter, as one or both vertices has migrated. By not doing this it would skew results for the next protocol cycle. + // We remove only the affected counters, as there could be other counters that 'this' silo has connections with another silo (which is not part of this exchange cycle). + _edgeWeights.Remove(edge); } + _logger.LogInformation("Removing transfer set from edge weights took {Elapsed}.", sw.Elapsed); - LogProtocolFinalized(idsToMigrate.Length); + LogProtocolFinalized(accepting.Length); } - private List<(SiloAddress Silo, ImmutableArray Candidates, long TransferScore)> CreateCandidateSets(ImmutableArray silos) + private List<(SiloAddress Silo, List Candidates, long TransferScore)> CreateCandidateSets(ImmutableArray silos) { - List<(SiloAddress Silo, ImmutableArray Candidates, long TransferScore)> candidateSets = new(silos.Length - 1); + List<(SiloAddress Silo, List Candidates, long TransferScore)> candidateSets = new(silos.Length - 1); var sw = ValueStopwatch.StartNew(); var localVertices = CreateLocalVertexEdges().ToList(); _logger.LogInformation("Computing local vertex edges took {Elapsed}.", sw.Elapsed); diff --git a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs index 885ac9ed16..3f050d4c27 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs @@ -3,6 +3,7 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Orleans.Placement.Rebalancing; @@ -22,8 +23,7 @@ internal sealed class CandidateVertexHeapElement(CandidateVertex value) : IHeapE public long AccumulatedTransferScore { get => Vertex.AccumulatedTransferScore; set => Vertex.AccumulatedTransferScore = value; } public VertexLocation Location { get; set; } int IHeapElement.HeapIndex { get; set; } - int IHeapElement.CompareTo(CandidateVertexHeapElement other) - => Vertex.AccumulatedTransferScore.CompareTo(other.Vertex.AccumulatedTransferScore); + int IHeapElement.CompareTo(CandidateVertexHeapElement other) => AccumulatedTransferScore.CompareTo(other.AccumulatedTransferScore); } internal interface IHeapElement where TElement : notnull @@ -90,7 +90,7 @@ public MaxHeap(ICollection items) _size = items.Count; var nodes = new TElement[_size]; items.CopyTo(nodes, 0); - for (var i = 0; i< nodes.Length; i++) + for (var i = 0; i < nodes.Length; i++) { nodes[i].HeapIndex = i; } @@ -111,6 +111,20 @@ public MaxHeap(ICollection items) /// public int Count => _size; + public TElement? FirstOrDefault() => _size > 0 ? _nodes[0] : default; + + public bool TryPeek([NotNullWhen(true)] out TElement value) + { + if (_size > 0) + { + value = _nodes[0]!; + return true; + } + + value = default!; + return false; + } + /// /// Returns the maximal element from the without removing it. /// From 4315acf92fd54cfd69eeb4ba6366b3193d7f6f19 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 20 May 2024 20:30:10 -0700 Subject: [PATCH 05/28] The breaking --- .../IActivationRebalancerSystemTarget.cs | 4 + .../Rebalancing/ActivationRebalancer.cs | 288 +++++++----------- .../Placement/Rebalancing/MaxHeap.cs | 11 +- .../Placement/Rebalancing/WeightedEdge.cs | 22 -- .../ActiveRebalancingTests/MaxHeapTests.cs | 2 +- 5 files changed, 126 insertions(+), 201 deletions(-) delete mode 100644 src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs diff --git a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs index a958713a1e..f980890818 100644 --- a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs +++ b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs @@ -60,6 +60,8 @@ static IActivationRebalancerSystemTarget GetReference(IGrainFactory grainFactory /// Returns a copy of this but with flipped sources and targets. /// public Edge Flip() => new(source: Target, target: Source); + + public override string ToString() => $"[{Source} -> {Target}]"; } /// @@ -87,6 +89,8 @@ static IActivationRebalancerSystemTarget GetReference(IGrainFactory grainFactory public bool Equals(EdgeVertex other) => Id == other.Id && Silo == other.Silo && IsMigratable == other.IsMigratable; public override int GetHashCode() => HashCode.Combine(Id, Silo, IsMigratable); + + public override string ToString() => $"[{Id}@{Silo}{(IsMigratable ? "" : "/NotMigratable")}]"; } /// diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 55fe68617b..ba190bf8bf 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -17,6 +17,7 @@ using Orleans.Runtime.Utilities; using System.Runtime.InteropServices; using System.Runtime.ExceptionServices; +using System.Diagnostics.CodeAnalysis; namespace Orleans.Runtime.Placement.Rebalancing; @@ -110,31 +111,18 @@ public async ValueTask TriggerExchangeRequest() var silos = _siloStatusOracle.GetActiveSilos(); if (silos.Length == 1) { - //_enableMessageSampling = false; LogSingleSiloCluster(); - return; // If its a single-silo cluster we have no business doing any kind of rebalancing + return; } else if (!_enableMessageSampling) { -// _enableMessageSampling = true; return; } - var sets = CreateCandidateSets(silos); - - var countWithNoExchangeSet = 0; - foreach (var set in sets) + foreach ((var candidateSilo, var offeredGrains, var _) in CreateCandidateSets(silos)) { - if (_currentExchangeSilo is not null) - { - // Skip this round if we are already in the process of exchanging with another silo. - return; - } - - (var candidateSilo, var offeredGrains, var _) = set; if (offeredGrains.Count == 0) { - countWithNoExchangeSet++; LogExchangeSetIsEmpty(candidateSilo); continue; } @@ -148,9 +136,7 @@ public async ValueTask TriggerExchangeRequest() LogBeginningProtocol(Silo, candidateSilo); var remoteRef = IActivationRebalancerSystemTarget.GetReference(_grainFactory, candidateSilo); var sw2 = ValueStopwatch.StartNew(); - _logger.LogInformation("Sending AcceptExchangeRequest"); - var response = await remoteRef.AcceptExchangeRequest(new (Silo, offeredGrains.ToImmutableArray(), GetLocalActivationCount())); - _logger.LogInformation("Sent AcceptExchangeRequest. It took {Elapsed}", sw2.Elapsed); + var response = await remoteRef.AcceptExchangeRequest(new(Silo, [.. offeredGrains], GetLocalActivationCount())); switch (response.Type) { @@ -179,44 +165,12 @@ public async ValueTask TriggerExchangeRequest() _currentExchangeSilo = null; } } - - /* - if (countWithNoExchangeSet == sets.Count) - { - // Disable message sampling for now, since there were no exchanges performed. - _logger.LogDebug("Placement has stabilized. Disabling sampling."); - _enableMessageSampling = false; - } - */ } private int GetLocalActivationCount() => _activationDirectory.Count + _activationCountOffset; - private struct AttachDebuggerOnFirstChance : IDisposable - { - public AttachDebuggerOnFirstChance() - { - AppDomain.CurrentDomain.FirstChanceException += CurrentDomain_FirstChanceException; - } - - public readonly void Dispose() - { - AppDomain.CurrentDomain.FirstChanceException -= CurrentDomain_FirstChanceException; - } - - private void CurrentDomain_FirstChanceException(object? sender, FirstChanceExceptionEventArgs e) - { - if (e.Exception is IndexOutOfRangeException) - { - Debugger.Launch(); - } - } - } - public async ValueTask AcceptExchangeRequest(AcceptExchangeRequest request) { - using var _ = new AttachDebuggerOnFirstChance(); - _logger.LogInformation("Received AcceptExchangeRequest from {Silo}", request.SendingSilo); if (request.SendingSilo.Equals(_currentExchangeSilo) && Silo.CompareTo(request.SendingSilo) <= 0) { @@ -232,7 +186,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha } var lastExchangeElapsed = _lastExchangedStopwatch.Elapsed; - if (lastExchangeElapsed < _options.RecoveryPeriod || _currentExchangeSilo != null) + if (lastExchangeElapsed < _options.RecoveryPeriod) { LogExchangedRecently(request.SendingSilo, lastExchangeElapsed, _options.RecoveryPeriod); return AcceptExchangeResponse.CachedExchangedRecently; @@ -302,30 +256,12 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha bool TryMigrateLocalToRemote() { - if (localHeap.Count == 0) - { - return false; - } - - var anticipatedImbalance = CalculateImbalance(localActivations - 1, remoteActivations + 1); - if (anticipatedImbalance > imbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + if (!TryMigrateCore(localHeap, localDelta: -1, remoteDelta: 1, out var chosenVertex)) { return false; } - var chosenVertex = localHeap.Pop(); - if (chosenVertex.AccumulatedTransferScore <= 0) - { - // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. - return false; - } - givingGrains.Add(chosenVertex.Id); - - localActivations--; - remoteActivations++; - imbalance = anticipatedImbalance; - foreach (var (connectedVertex, transferScore) in chosenVertex.ConnectedVertices) { switch (connectedVertex.Location) @@ -351,29 +287,12 @@ bool TryMigrateLocalToRemote() bool TryMigrateRemoteToLocal() { - if (remoteHeap.Count == 0) + if (!TryMigrateCore(remoteHeap, localDelta: 1, remoteDelta: -1, out var chosenVertex)) { return false; } - var anticipatedImbalance = CalculateImbalance(localActivations + 1, remoteActivations - 1); - if (anticipatedImbalance > imbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) - { - return false; - } - - var chosenVertex = remoteHeap.Pop(); - if (chosenVertex.AccumulatedTransferScore <= 0) - { - // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. - return false; - } - acceptedGrains.Add(chosenVertex.Id); - - localActivations++; - remoteActivations--; - imbalance = anticipatedImbalance; foreach (var (connectedVertex, transferScore) in chosenVertex.ConnectedVertices) { switch (connectedVertex.Location) @@ -397,7 +316,33 @@ bool TryMigrateRemoteToLocal() return true; } - static int CalculateImbalance(int left, int right) => Math.Abs(Math.Abs(left) - Math.Abs(right)); + bool TryMigrateCore(MaxHeap sourceHeap, int localDelta, int remoteDelta, [NotNullWhen(true)] out CandidateVertexHeapElement? chosenVertex) + { + chosenVertex = null; + if (sourceHeap.Count == 0) + { + return false; + } + + var anticipatedImbalance = CalculateImbalance(localActivations + localDelta, remoteActivations + remoteDelta); + if (anticipatedImbalance > imbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + { + return false; + } + + chosenVertex = sourceHeap.Pop(); + if (chosenVertex.AccumulatedTransferScore <= 0) + { + // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. + return false; + } + + localActivations += localDelta; + remoteActivations += remoteDelta; + imbalance = anticipatedImbalance; + return true; + } + } catch (Exception exception) { @@ -408,60 +353,61 @@ bool TryMigrateRemoteToLocal() { _currentExchangeSilo = null; } + } - (MaxHeap LocalHeap, MaxHeap RemoteHeap) CreateCandidateHeaps(List localSet, ImmutableArray remoteSet) + private static int CalculateImbalance(int left, int right) => Math.Abs(Math.Abs(left) - Math.Abs(right)); + private static (MaxHeap Local, MaxHeap Remote) CreateCandidateHeaps(List local, ImmutableArray remote) + { + Dictionary sourceIndex = new(local.Count + remote.Length); + foreach (var element in local) { - Dictionary sourceIndex = []; - foreach (var element in localSet) - { - sourceIndex[element.Id] = element; - } + sourceIndex[element.Id] = element; + } - foreach (var element in remoteSet) - { - sourceIndex[element.Id] = element; - } + foreach (var element in remote) + { + sourceIndex[element.Id] = element; + } - Dictionary index = []; - List localVertexList = []; - foreach (var element in localSet) - { - var vertex = CreateVertex(sourceIndex, index, element); - vertex.Location = VertexLocation.Local; - localVertexList.Add(vertex); - } + Dictionary index = []; + List localVertexList = new(local.Count); + foreach (var element in local) + { + var vertex = CreateVertex(sourceIndex, index, element); + vertex.Location = VertexLocation.Local; + localVertexList.Add(vertex); + } - List remoteVertexList = []; - foreach (var element in remoteSet) - { - var vertex = CreateVertex(sourceIndex, index, element); - vertex.Location = VertexLocation.Remote; - remoteVertexList.Add(vertex); - } + List remoteVertexList = new(remote.Length); + foreach (var element in remote) + { + var vertex = CreateVertex(sourceIndex, index, element); + vertex.Location = VertexLocation.Remote; + remoteVertexList.Add(vertex); + } - var localHeap = new MaxHeap(localVertexList); - var remoteHeap = new MaxHeap(remoteVertexList); - return (localHeap, remoteHeap); + var localHeap = new MaxHeap(localVertexList); + var remoteHeap = new MaxHeap(remoteVertexList); + return (localHeap, remoteHeap); - static CandidateVertexHeapElement CreateVertex(Dictionary sourceIndex, Dictionary index, CandidateVertex element) + static CandidateVertexHeapElement CreateVertex(Dictionary sourceIndex, Dictionary index, CandidateVertex element) + { + var vertex = GetOrAddVertex(index, element); + foreach (var connectedVertex in element.ConnectedVertices) { - var vertex = GetOrAddVertex(index, element); - foreach (var connectedVertex in element.ConnectedVertices) + if (sourceIndex.TryGetValue(connectedVertex.Id, out var connected)) { - if (sourceIndex.TryGetValue(connectedVertex.Id, out var connected)) - { - vertex.ConnectedVertices.Add((GetOrAddVertex(index, connected), connectedVertex.TransferScore)); - } + vertex.ConnectedVertices.Add((GetOrAddVertex(index, connected), connectedVertex.TransferScore)); } + } - return vertex; + return vertex; - static CandidateVertexHeapElement GetOrAddVertex(Dictionary index, CandidateVertex element) - { - ref var vertex = ref CollectionsMarshal.GetValueRefOrAddDefault(index, element.Id, out var exists); - vertex ??= new(element); - return vertex; - } + static CandidateVertexHeapElement GetOrAddVertex(Dictionary index, CandidateVertex element) + { + ref var vertex = ref CollectionsMarshal.GetValueRefOrAddDefault(index, element.Id, out var exists); + vertex ??= new(element); + return vertex; } } } @@ -604,13 +550,15 @@ private List CreateCandidateSet(IEnumerable edges, foreach (var entry in grouping) { - if (entry.Direction == Direction.LocalToLocal) + if (entry.Direction is Direction.LocalToLocal) { // Since its L2L, it means the partner silo will be 'this' silo, so we don't need to filter by the partner silo. accLocalScore += entry.Weight; } - else if (entry.TargetSilo.Equals(otherSilo) && entry.Direction is Direction.RemoteToLocal or Direction.LocalToRemote) + else if (entry.PartnerSilo.Equals(otherSilo)) { + Debug.Assert(entry.Direction is Direction.RemoteToLocal or Direction.LocalToRemote); + // We need to filter here by 'otherSilo' since any L2R or R2L edge can be between the current vertex and a vertex in a silo that is not in 'otherSilo'. accRemoteScore += entry.Weight; } @@ -659,7 +607,7 @@ private IEnumerable CreateLocalVertexEdges() continue; } - var vertexEdge = CreateVertexEdge(new WeightedEdge(edge, count)); + var vertexEdge = CreateVertexEdge(edge, count); yield return vertexEdge; if (vertexEdge.Direction == Direction.LocalToLocal) @@ -668,56 +616,48 @@ private IEnumerable CreateLocalVertexEdges() // Once an edge exists it means 2 grains are temporally linked, this means that there is a cost associated to potentially move either one of them. // Since the construction of the candidate set takes into account also local connection (which increases the cost of migration), we need // to take into account the edge not only from a source's perspective, but also the target's one, as it too will take part on the candidate set. - var flippedEdge = CreateVertexEdge(new WeightedEdge(edge.Flip(), count)); + var flippedEdge = CreateVertexEdge(edge.Flip(), count); yield return flippedEdge; } } - VertexEdge CreateVertexEdge(in WeightedEdge counter) + VertexEdge CreateVertexEdge(in Edge edge, long weight) { - var direction = Direction.Unspecified; - - direction = IsSourceThisSilo(counter) - ? IsTargetThisSilo(counter) ? Direction.LocalToLocal : Direction.LocalToRemote - : Direction.RemoteToLocal; - - Debug.Assert(direction != Direction.Unspecified); // this can only occur when both: source and target are remote (which can not happen) - - return direction switch + return (IsSourceLocal(edge), IsTargetLocal(edge)) switch { - Direction.LocalToLocal => new( - SourceId: counter.Edge.Source.Id, // 'local' vertex was the 'source' of the communication - TargetId: counter.Edge.Target.Id, - IsMigratable: counter.Edge.Source.IsMigratable, - TargetSilo: Silo, // the partner was 'local' (note: this.Silo = Source.Silo = Target.Silo) - Direction: direction, - Weight: counter.Weight), - - Direction.LocalToRemote => new( - SourceId: counter.Edge.Source.Id, // 'local' vertex was the 'source' of the communication - TargetId: counter.Edge.Target.Id, - IsMigratable: counter.Edge.Source.IsMigratable, - TargetSilo: counter.Edge.Target.Silo, // the partner was 'remote' - Direction: direction, - Weight: counter.Weight), - - Direction.RemoteToLocal => new( - SourceId: counter.Edge.Target.Id, // 'local' vertex was the 'target' of the communication - TargetId: counter.Edge.Source.Id, - IsMigratable: counter.Edge.Target.IsMigratable, - TargetSilo: counter.Edge.Source.Silo, // the partner was 'remote' - Direction: direction, - Weight: counter.Weight), - - _ => throw new UnreachableException($"The edge direction {direction} is out of range.") + (true, true) => new( + SourceId: edge.Source.Id, // 'local' vertex was the 'source' of the communication + TargetId: edge.Target.Id, + IsMigratable: edge.Source.IsMigratable, + PartnerSilo: Silo, // the partner was 'local' (note: this.Silo = Source.Silo = Target.Silo) + Direction: Direction.LocalToLocal, + Weight: weight), + + (true, false) => new( + SourceId: edge.Source.Id, // 'local' vertex was the 'source' of the communication + TargetId: edge.Target.Id, + IsMigratable: edge.Source.IsMigratable, + PartnerSilo: edge.Target.Silo, // the partner was 'remote' + Direction: Direction.LocalToRemote, + Weight: weight), + + (false, true) => new( + SourceId: edge.Target.Id, // 'local' vertex was the 'target' of the communication + TargetId: edge.Source.Id, + IsMigratable: edge.Target.IsMigratable, + PartnerSilo: edge.Source.Silo, // the partner was 'remote' + Direction: Direction.RemoteToLocal, + Weight: weight), + + _ => throw new UnreachableException($"The edge {edge} has an invalid source and target: neither refer to the local silo.") }; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool IsSourceThisSilo(in WeightedEdge counter) => counter.Edge.Source.Silo.IsSameLogicalSilo(Silo); + bool IsSourceLocal(in Edge edge) => edge.Source.Silo.IsSameLogicalSilo(Silo); [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool IsTargetThisSilo(in WeightedEdge counter) => counter.Edge.Target.Silo.IsSameLogicalSilo(Silo); + bool IsTargetLocal(in Edge edge) => edge.Target.Silo.IsSameLogicalSilo(Silo); } public void Participate(ISiloLifecycle observer) @@ -764,8 +704,8 @@ private enum Direction : byte /// The id of the grain it represents. /// The id of the connected vertex (the one the communication took place with). /// Specifies if the vertex with is a migratable type. - /// The silo partner which interacted with the silo of vertex with . + /// The silo partner which interacted with the silo of vertex with . /// The edge's direction /// The number of estimated messages exchanged between and . - private readonly record struct VertexEdge(GrainId SourceId, GrainId TargetId, bool IsMigratable, SiloAddress TargetSilo, Direction Direction, long Weight); + private readonly record struct VertexEdge(GrainId SourceId, GrainId TargetId, bool IsMigratable, SiloAddress PartnerSilo, Direction Direction, long Weight); } \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs index 3f050d4c27..c88bc8c19e 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs @@ -83,16 +83,19 @@ static MaxHeap() /// Constructs the heap using a heapify operation, /// which is generally faster than enqueuing individual elements sequentially. /// - public MaxHeap(ICollection items) + public MaxHeap(List items) { ArgumentNullException.ThrowIfNull(items); _size = items.Count; var nodes = new TElement[_size]; - items.CopyTo(nodes, 0); - for (var i = 0; i < nodes.Length; i++) + + var i = 0; + foreach (var item in items) { - nodes[i].HeapIndex = i; + nodes[i] = item; + item.HeapIndex = i; + i++; } _nodes = nodes; diff --git a/src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs b/src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs deleted file mode 100644 index 47566f94fd..0000000000 --- a/src/Orleans.Runtime/Placement/Rebalancing/WeightedEdge.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Diagnostics; -using Orleans.Placement.Rebalancing; - -namespace Orleans.Runtime.Placement.Rebalancing; - -#nullable enable - -/// -/// Represents a weighted . -/// -[DebuggerDisplay("Value {Value} | Edge = {Edge}")] -internal readonly struct WeightedEdge(Edge edge, long weight) -{ - public readonly Edge Edge = edge; - - public readonly long Weight = weight; - - /// - /// Returns a copy of this but with flipped sources and targets. - /// - public WeightedEdge Flip() => new(Edge.Flip(), Weight); -} \ No newline at end of file diff --git a/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs b/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs index 1f3600e02b..af198ddbb5 100644 --- a/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs @@ -25,7 +25,7 @@ public void HeapPropertyIsMaintained() } Random.Shared.Shuffle(edges); - var heap = new MaxHeap(edges); + var heap = new MaxHeap([.. edges]); Assert.Equal(100, heap.Count); Assert.Equal(99, heap.Peek().Value); Assert.Equal(99, heap.Peek().Value); From f79d6e734aabb5b97704be800a75f82b33587384 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 20 May 2024 20:59:18 -0700 Subject: [PATCH 06/28] Fix & clean up --- .../Rebalancing/ActivationRebalancer.cs | 89 ++++++++++--------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index ba190bf8bf..78a9a5d106 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -202,7 +202,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha var givingGrains = ImmutableArray.CreateBuilder(); var remoteSet = request.ExchangeSet; _logger.LogInformation("About to create candidate set"); - var localSet = CreateCandidateSet(CreateLocalVertexEdges(), request.SendingSilo); + var localSet = GetCandidatesForSilo(GetMigrationCandidates(), request.SendingSilo); _logger.LogInformation("Created candidate set"); // We need to determine 2 subsets: @@ -230,7 +230,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha // If more is gained by giving grains to the remote silo than taking from it, we will try giving first. var localScore = localHeap.FirstOrDefault()?.AccumulatedTransferScore ?? 0; var remoteScore = remoteHeap.FirstOrDefault()?.AccumulatedTransferScore ?? 0; - if (localScore > remoteScore || localActivations > remoteActivations) + if (localScore > remoteScore || localScore == remoteScore && localActivations > remoteActivations) { if (TryMigrateLocalToRemote()) continue; if (TryMigrateRemoteToLocal()) continue; @@ -507,22 +507,22 @@ private List<(SiloAddress Silo, List Candidates, long TransferS { List<(SiloAddress Silo, List Candidates, long TransferScore)> candidateSets = new(silos.Length - 1); var sw = ValueStopwatch.StartNew(); - var localVertices = CreateLocalVertexEdges().ToList(); + var localCandidates = GetMigrationCandidates(); _logger.LogInformation("Computing local vertex edges took {Elapsed}.", sw.Elapsed); sw.Restart(); foreach (var siloAddress in silos) { - if (siloAddress.IsSameLogicalSilo(Silo)) + if (siloAddress.Equals(Silo)) { // We aren't going to exchange anything with ourselves, so skip this silo. continue; } - var candidates = CreateCandidateSet(localVertices, siloAddress); - var totalAccTransferScore = candidates.Sum(x => x.AccumulatedTransferScore); + var candidatesForRemote = GetCandidatesForSilo(localCandidates, siloAddress); + var totalAccTransferScore = candidatesForRemote.Sum(x => x.AccumulatedTransferScore); - candidateSets.Add(new(siloAddress, [.. candidates], totalAccTransferScore)); + candidateSets.Add(new(siloAddress, [.. candidatesForRemote], totalAccTransferScore)); } _logger.LogInformation("Computing candidate set per-silo took {Elapsed}.", sw.Elapsed); @@ -533,34 +533,32 @@ private List<(SiloAddress Silo, List Candidates, long TransferS return candidateSets; } - private List CreateCandidateSet(IEnumerable edges, SiloAddress otherSilo) + private List GetCandidatesForSilo(List> migrationCandidates, SiloAddress otherSilo) { - Debug.Assert(otherSilo.IsSameLogicalSilo(Silo) is false); + Debug.Assert(!otherSilo.Equals(Silo)); - List candidates = []; + List result = []; // We skip types that cant be migrated. Instead the same edge will be recorded from the receiver, so its hosting silo will add it as a candidate to be migrated (over here). // We are sure that the receiver is an migratable grain, because the gateway forbids edges that have non-migratable vertices on both sides. - foreach (var grouping in edges - .Where(x => x.IsMigratable) - .GroupBy(x => x.SourceId)) + foreach (var grainEdges in migrationCandidates) { var accLocalScore = 0L; var accRemoteScore = 0L; - foreach (var entry in grouping) + foreach (var edge in grainEdges) { - if (entry.Direction is Direction.LocalToLocal) + if (edge.Direction is Direction.LocalToLocal) { // Since its L2L, it means the partner silo will be 'this' silo, so we don't need to filter by the partner silo. - accLocalScore += entry.Weight; + accLocalScore += edge.Weight; } - else if (entry.PartnerSilo.Equals(otherSilo)) + else if (edge.PartnerSilo.Equals(otherSilo)) { - Debug.Assert(entry.Direction is Direction.RemoteToLocal or Direction.LocalToRemote); + Debug.Assert(edge.Direction is Direction.RemoteToLocal or Direction.LocalToRemote); // We need to filter here by 'otherSilo' since any L2R or R2L edge can be between the current vertex and a vertex in a silo that is not in 'otherSilo'. - accRemoteScore += entry.Weight; + accRemoteScore += edge.Weight; } } @@ -572,28 +570,30 @@ private List CreateCandidateSet(IEnumerable edges, } var connVertices = ImmutableArray.CreateBuilder(); - foreach (var x in grouping) + foreach (var edge in grainEdges) { // Note that the connected vertices can be of types which are not migratable, it is important to keep them, // as they too impact the migration cost of the current candidate vertex, especially if they are local to the candidate // as those calls would be potentially converted to remote calls, after the migration of the current candidate. // 'Weight' here represent the weight of a single edge, not the accumulated like above. - connVertices.Add(new CandidateConnectedVertex(x.TargetId, x.Weight)); + connVertices.Add(new CandidateConnectedVertex(edge.TargetId, edge.Weight)); } CandidateVertex candidate = new() { - Id = grouping.Key, + Id = grainEdges.Key, AccumulatedTransferScore = totalAccScore, ConnectedVertices = connVertices.ToImmutable() }; - candidates.Add(candidate); + result.Add(candidate); } - return candidates; + return result; } + private List> GetMigrationCandidates() => CreateLocalVertexEdges().Where(x => x.IsMigratable).GroupBy(x => x.SourceId).ToList(); + /// /// Creates a collection of 'local' vertex edges. Multiple entries can have the same Id. /// @@ -608,6 +608,12 @@ private IEnumerable CreateLocalVertexEdges() } var vertexEdge = CreateVertexEdge(edge, count); + if (vertexEdge.Direction is Direction.Unspecified) + { + // This can occur when a message is re-routed via this silo. + continue; + } + yield return vertexEdge; if (vertexEdge.Direction == Direction.LocalToLocal) @@ -625,39 +631,36 @@ VertexEdge CreateVertexEdge(in Edge edge, long weight) { return (IsSourceLocal(edge), IsTargetLocal(edge)) switch { - (true, true) => new( - SourceId: edge.Source.Id, // 'local' vertex was the 'source' of the communication - TargetId: edge.Target.Id, - IsMigratable: edge.Source.IsMigratable, - PartnerSilo: Silo, // the partner was 'local' (note: this.Silo = Source.Silo = Target.Silo) - Direction: Direction.LocalToLocal, - Weight: weight), - + (true, true) => new( + SourceId: edge.Source.Id, // 'local' vertex was the 'source' of the communication + TargetId: edge.Target.Id, + IsMigratable: edge.Source.IsMigratable, + PartnerSilo: Silo, // the partner was 'local' (note: this.Silo = Source.Silo = Target.Silo) + Direction: Direction.LocalToLocal, + Weight: weight), (true, false) => new( - SourceId: edge.Source.Id, // 'local' vertex was the 'source' of the communication + SourceId: edge.Source.Id, // 'local' vertex was the 'source' of the communication TargetId: edge.Target.Id, IsMigratable: edge.Source.IsMigratable, PartnerSilo: edge.Target.Silo, // the partner was 'remote' Direction: Direction.LocalToRemote, Weight: weight), - (false, true) => new( - SourceId: edge.Target.Id, // 'local' vertex was the 'target' of the communication + SourceId: edge.Target.Id, // 'local' vertex was the 'target' of the communication TargetId: edge.Source.Id, IsMigratable: edge.Target.IsMigratable, PartnerSilo: edge.Source.Silo, // the partner was 'remote' Direction: Direction.RemoteToLocal, Weight: weight), - - _ => throw new UnreachableException($"The edge {edge} has an invalid source and target: neither refer to the local silo.") + _ => default }; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool IsSourceLocal(in Edge edge) => edge.Source.Silo.IsSameLogicalSilo(Silo); + bool IsSourceLocal(in Edge edge) => edge.Source.Silo.Equals(Silo); [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool IsTargetLocal(in Edge edge) => edge.Target.Silo.IsSameLogicalSilo(Silo); + bool IsTargetLocal(in Edge edge) => edge.Target.Silo.Equals(Silo); } public void Participate(ISiloLifecycle observer) @@ -692,10 +695,10 @@ void ISiloStatusListener.SiloStatusChangeNotification(SiloAddress updatedSilo, S private enum Direction : byte { - Unspecified = 0, - LocalToLocal = 1, - LocalToRemote = 2, - RemoteToLocal = 3 + Unspecified, + LocalToLocal, + LocalToRemote, + RemoteToLocal } /// From 51f7608298165f027f2fd6f589c9184bba65374c Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 20 May 2024 21:33:59 -0700 Subject: [PATCH 07/28] wip --- .../Options/ActiveRebalancingOptions.cs | 10 +--------- .../Rebalancing/ActivationRebalancer.cs | 20 ++++++++++--------- .../Placement/Rebalancing/MaxHeap.cs | 12 +++++++++++ 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index 3c3596a129..9491249554 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -20,15 +20,7 @@ public sealed class ActiveRebalancingOptions /// In order to preserve memory, the most heaviest links are recorded in a probabilistic way, so there is an inherent error associated with that. /// That error is inversely proportional to this value, so values under 100 are not recommended. If you notice that the system is not converging fast enough, do consider increasing this number. /// - public uint MaxEdgeCount { get; set; } = - - - - 10 * - - - - DEFAULT_MAX_EDGE_COUNT; + public uint MaxEdgeCount { get; set; } = DEFAULT_MAX_EDGE_COUNT; /// /// The default value of . diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 78a9a5d106..a483a0496f 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -211,7 +211,8 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha var remoteActivations = request.ActivationCountSnapshot; var localActivations = GetLocalActivationCount(); - var imbalance = CalculateImbalance(remoteActivations, localActivations); + var initialImbalance = CalculateImbalance(remoteActivations, localActivations); + int imbalance = initialImbalance; _logger.LogInformation("Imbalance is {Imbalance} (remote: {RemoteCount} vs local {LocalCount})", imbalance, remoteActivations, localActivations); var (localHeap, remoteHeap) = CreateCandidateHeaps(localSet, remoteSet); @@ -318,22 +319,23 @@ bool TryMigrateRemoteToLocal() bool TryMigrateCore(MaxHeap sourceHeap, int localDelta, int remoteDelta, [NotNullWhen(true)] out CandidateVertexHeapElement? chosenVertex) { - chosenVertex = null; - if (sourceHeap.Count == 0) + var anticipatedImbalance = CalculateImbalance(localActivations + localDelta, remoteActivations + remoteDelta); + if (anticipatedImbalance >= initialImbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) { + // Taking from this heap would not improve imbalance. + chosenVertex = null; return false; } - var anticipatedImbalance = CalculateImbalance(localActivations + localDelta, remoteActivations + remoteDelta); - if (anticipatedImbalance > imbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + if (!sourceHeap.TryPop(out chosenVertex)) { + // Heap is empty. return false; } - chosenVertex = sourceHeap.Pop(); - if (chosenVertex.AccumulatedTransferScore <= 0) + if (chosenVertex.AccumulatedTransferScore < 0) { - // If it got affected by a previous run, and the score is not positive, simply pop and ignore it. + // If it got affected by a previous run, and the score is negative, simply pop and ignore it. return false; } @@ -355,7 +357,7 @@ bool TryMigrateCore(MaxHeap sourceHeap, int localDel } } - private static int CalculateImbalance(int left, int right) => Math.Abs(Math.Abs(left) - Math.Abs(right)); + private static int CalculateImbalance(int left, int right) => (int)Math.Abs(Math.Abs((long)left) - Math.Abs((long)right)); private static (MaxHeap Local, MaxHeap Remote) CreateCandidateHeaps(List local, ImmutableArray remote) { Dictionary sourceIndex = new(local.Count + remote.Length); diff --git a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs index c88bc8c19e..270dac03e2 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs @@ -143,6 +143,18 @@ public TElement Peek() return _nodes[0]!; } + public bool TryPop([NotNullWhen(true)] out TElement value) + { + if (_size > 0) + { + value = Pop(); + return true; + } + + value = default!; + return false; + } + /// /// Removes and returns the maximal element from the . /// From e463a274a4845eacc31953d424750ad1be229a4a Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Tue, 21 May 2024 07:31:42 -0700 Subject: [PATCH 08/28] Log 1st chance exceptions in fan-out test --- test/Benchmarks/Ping/FanoutBenchmark.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/Benchmarks/Ping/FanoutBenchmark.cs b/test/Benchmarks/Ping/FanoutBenchmark.cs index 29e1e0f448..b2dcf79c2f 100644 --- a/test/Benchmarks/Ping/FanoutBenchmark.cs +++ b/test/Benchmarks/Ping/FanoutBenchmark.cs @@ -90,6 +90,8 @@ public FanoutBenchmark(int numSilos, bool startClient, bool grainsOnSecondariesO _onCancelEvent = CancelPressed; Console.CancelKeyPress += _onCancelEvent; + AppDomain.CurrentDomain.FirstChanceException += (object sender, System.Runtime.ExceptionServices.FirstChanceExceptionEventArgs e) => Console.WriteLine("FIRST CHANCE EXCEPTION: " + LogFormatter.PrintException(e.Exception)); + AppDomain.CurrentDomain.UnhandledException += (object sender, UnhandledExceptionEventArgs e) => Console.WriteLine("UNHANDLED EXCEPTION: " + LogFormatter.PrintException((Exception)e.ExceptionObject)); } private void CancelPressed(object sender, ConsoleCancelEventArgs e) From cc7bfab8fb91c3054a23e1687d08adaab530605a Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Tue, 21 May 2024 22:00:38 -0700 Subject: [PATCH 09/28] Fixes --- .../IActivationRebalancerSystemTarget.cs | 19 +++++- .../Rebalancing/ActivationRebalancer.Log.cs | 4 +- .../Rebalancing/ActivationRebalancer.cs | 61 +++++++++---------- .../Placement/Rebalancing/MaxHeap.cs | 4 +- .../CustomToleranceTests.cs | 3 +- .../DefaultToleranceTests.cs | 2 +- 6 files changed, 52 insertions(+), 41 deletions(-) diff --git a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs index f980890818..a977fef48d 100644 --- a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs +++ b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs @@ -117,12 +117,17 @@ internal sealed class CandidateVertex /// These will be important when this vertex is removed from the max-sorted heap on the receiver silo. [Id(2), Immutable] public ImmutableArray ConnectedVertices { get; init; } = []; + + public override string ToString() => $"[{Id} * {AccumulatedTransferScore} -> [{string.Join(", ", ConnectedVertices)}]]"; } [GenerateSerializer, Immutable] public readonly struct CandidateConnectedVertex(GrainId id, long transferScore) { + [Id(0)] public GrainId Id { get; } = id; + + [Id(1)] public long TransferScore { get; } = transferScore; public static bool operator ==(CandidateConnectedVertex left, CandidateConnectedVertex right) => left.Equals(right); @@ -132,6 +137,8 @@ internal sealed class CandidateVertex public bool Equals(CandidateConnectedVertex other) => Id == other.Id && TransferScore == other.TransferScore; public override int GetHashCode() => HashCode.Combine(Id, TransferScore); + + public override string ToString() => $"[{Id} * {TransferScore}]"; } [GenerateSerializer, Immutable] @@ -165,9 +172,15 @@ internal sealed class AcceptExchangeResponse(AcceptExchangeResponse.ResponseType [Id(0)] public ResponseType Type { get; } = type; + /// + /// The grains which the sender is asking the receiver to transfer. + /// [Id(1)] public ImmutableArray AcceptedGrainIds { get; } = acceptedGrains; + /// + /// The grains which the receiver is transferring to the sender. + /// [Id(2)] public ImmutableArray GivenGrainIds { get; } = givenGrains; @@ -177,16 +190,16 @@ public enum ResponseType /// /// The exchange was accepted and an exchange set is returned. /// - Success = 0, + Success, /// /// The other silo has been recently involved in another exchange. /// - ExchangedRecently = 1, + ExchangedRecently, /// /// An attempt to do an exchange between this and the other silo was about to happen at the same time. /// - MutualExchangeAttempt = 2 + MutualExchangeAttempt } } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs index 06d296b179..e3ecf4ba5a 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs @@ -29,8 +29,8 @@ internal partial class ActivationRebalancer [LoggerMessage(Level = LogLevel.Debug, Message = "Exchange request from {ThisSilo} superseded by a mutual exchange attempt with {CandidateSilo}.")] private partial void LogMutualExchangeAttemptResponse(SiloAddress thisSilo, SiloAddress candidateSilo); - [LoggerMessage(Level = LogLevel.Debug, Message = "I have successfully finalized my part of the exchange protocol. It was decided that I will take on a total of {ActivationCount} activations.")] - private partial void LogProtocolFinalized(int activationCount); + [LoggerMessage(Level = LogLevel.Debug, Message = "I have successfully finalized my part of the exchange protocol. It was decided that I will give {GivingActivationCount} activations and take on a total of {TakingActivationCount} activations.")] + private partial void LogProtocolFinalized(int givingActivationCount, int takingActivationCount); [LoggerMessage(Level = LogLevel.Warning, Message = "An error occurred while performing exchange request from {ThisSilo} to {CandidateSilo}. I will try the next best candidate (if one is available), otherwise I will wait for my next period to come.")] private partial void LogErrorOnProtocolExecution(Exception exception, SiloAddress thisSilo, SiloAddress candidateSilo); diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index a483a0496f..d7a20b93da 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -16,7 +16,6 @@ using Orleans.Configuration; using Orleans.Runtime.Utilities; using System.Runtime.InteropServices; -using System.Runtime.ExceptionServices; using System.Diagnostics.CodeAnalysis; namespace Orleans.Runtime.Placement.Rebalancing; @@ -142,7 +141,7 @@ public async ValueTask TriggerExchangeRequest() { case AcceptExchangeResponse.ResponseType.Success: // Exchange was successful, no need to iterate over another candidate. - await FinalizeProtocol(response.AcceptedGrainIds, response.GivenGrainIds, isReceiver: false, candidateSilo); + await FinalizeProtocol(response.AcceptedGrainIds, response.GivenGrainIds, candidateSilo); return; case AcceptExchangeResponse.ResponseType.ExchangedRecently: // The remote silo has been recently involved in another exchange, try the next best candidate. @@ -250,7 +249,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha swTxs.Restart(); var giving = givingGrains.ToImmutable(); var accepting = acceptedGrains.ToImmutable(); - await FinalizeProtocol(giving, accepting, isReceiver: true, request.SendingSilo); + await FinalizeProtocol(giving, accepting, request.SendingSilo); _logger.LogInformation("Finalizing protocol based on provided set took {Elapsed}", swTxs.Elapsed); return new(AcceptExchangeResponse.ResponseType.Success, accepting, giving); @@ -333,9 +332,9 @@ bool TryMigrateCore(MaxHeap sourceHeap, int localDel return false; } - if (chosenVertex.AccumulatedTransferScore < 0) + if (chosenVertex.AccumulatedTransferScore <= 0) { - // If it got affected by a previous run, and the score is negative, simply pop and ignore it. + // If it got affected by a previous run, and the score is zero or negative, simply pop and ignore it. return false; } @@ -401,6 +400,10 @@ static CandidateVertexHeapElement CreateVertex(Dictionary /// /// Initiates the actual migration process of to 'this' silo. - /// If it proceeds to update . /// Updates the affected counters within to reflect all . /// /// /// The grain ids to migrate to the remote host. /// The grain ids to which are migrating to the local host. - /// Is the caller, the protocol receiver or not. - private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArray accepting, bool isReceiver, SiloAddress targetSilo) + private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArray accepting, SiloAddress targetSilo) { - if (giving.Length == 0) - { - LogProtocolFinalized(giving.Length); - return; - } - // The protocol concluded that 'this' silo should take on 'set', so we hint to the director accordingly. RequestContext.Set(IPlacementDirector.PlacementHintKey, targetSilo); List migrationTasks = []; @@ -457,11 +452,6 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr } _logger.LogInformation("Waiting for {Count} grains to migrate took {Elapsed}", giving.Length, sw1.Elapsed); - if (isReceiver) - { - // Stamp this silos exchange for a potential next pair exchange request. - _lastExchangedStopwatch.Restart(); - } // Avoid mutating the source while enumerating it. var sw = ValueStopwatch.StartNew(); @@ -479,30 +469,35 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr affected.Add(id); } - foreach (var (edge, count, error) in _edgeWeights.Elements) + if (affected.Count > 0) { - if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id)) + foreach (var (edge, count, error) in _edgeWeights.Elements) { - toRemove.Add(edge); + if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id)) + { + toRemove.Add(edge); + } } - } - foreach (var edge in toRemove) - { - if (++iterations % 128 == 0) + foreach (var edge in toRemove) { - // Give other tasks a chance to execute periodically. - await Task.Delay(1); - } + if (++iterations % 128 == 0) + { + // Give other tasks a chance to execute periodically. + await Task.Delay(1); + } - // Totally remove this counter, as one or both vertices has migrated. By not doing this it would skew results for the next protocol cycle. - // We remove only the affected counters, as there could be other counters that 'this' silo has connections with another silo (which is not part of this exchange cycle). - _edgeWeights.Remove(edge); + // Totally remove this counter, as one or both vertices has migrated. By not doing this it would skew results for the next protocol cycle. + // We remove only the affected counters, as there could be other counters that 'this' silo has connections with another silo (which is not part of this exchange cycle). + _edgeWeights.Remove(edge); + } } _logger.LogInformation("Removing transfer set from edge weights took {Elapsed}.", sw.Elapsed); - LogProtocolFinalized(accepting.Length); + // Stamp this silos exchange for a potential next pair exchange request. + _lastExchangedStopwatch.Restart(); + LogProtocolFinalized(giving.Length, accepting.Length); } private List<(SiloAddress Silo, List Candidates, long TransferScore)> CreateCandidateSets(ImmutableArray silos) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs index 270dac03e2..02fecd330d 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs @@ -11,10 +11,12 @@ namespace Orleans.Runtime.Placement.Rebalancing; internal enum VertexLocation { + Unknown, Local, Remote } +[DebuggerDisplay("{Vertex} @ {Location}")] internal sealed class CandidateVertexHeapElement(CandidateVertex value) : IHeapElement { public CandidateVertex Vertex { get; } = value; @@ -22,7 +24,7 @@ internal sealed class CandidateVertexHeapElement(CandidateVertex value) : IHeapE public GrainId Id => Vertex.Id; public long AccumulatedTransferScore { get => Vertex.AccumulatedTransferScore; set => Vertex.AccumulatedTransferScore = value; } public VertexLocation Location { get; set; } - int IHeapElement.HeapIndex { get; set; } + int IHeapElement.HeapIndex { get; set; } = -1; int IHeapElement.CompareTo(CandidateVertexHeapElement other) => AccumulatedTransferScore.CompareTo(other.AccumulatedTransferScore); } diff --git a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs index 91cd69cbb8..b1f7f08d61 100644 --- a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs @@ -1,3 +1,4 @@ +using System.ComponentModel; using Microsoft.Extensions.DependencyInjection; using Orleans.Configuration; using Orleans.Placement; @@ -11,7 +12,7 @@ namespace UnitTests.ActiveRebalancingTests; -[TestCategory("Functional"), TestCategory("ActiveRebalancing")] +[TestCategory("Functional"), TestCategory("ActiveRebalancing"), Category("BVT")] public class CustomToleranceTests(CustomToleranceTests.Fixture fixture) : RebalancingTestBase(fixture), IClassFixture { [Fact] diff --git a/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs index 7dd8ec5741..9795f0a6c4 100644 --- a/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs @@ -224,7 +224,7 @@ public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOthe } // Since A moved to silo 2 at this point, it will be twice as strongly connected to C as it is to B, - // even though its now making remote calls (to B)! Thats why we trigger the exchange from 's1_rebalancer' + // even though its now making remote calls (to B)! That's why we trigger the exchange from 's1_rebalancer' await Silo1Rebalancer.TriggerExchangeRequest(); do From ba21e6b5cd48ecede9597aa7e75de85172ee3182 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Tue, 21 May 2024 22:16:17 -0700 Subject: [PATCH 10/28] Reschedule timer whether sending or receiving a request --- .../Placement/Rebalancing/ActivationRebalancer.cs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index d7a20b93da..15434d5d46 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -71,8 +71,8 @@ private Task OnActiveStart(CancellationToken cancellationToken) { Scheduler.QueueAction(() => { - var dueTime = RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime); - RegisterOrUpdateTimer(dueTime); + // Schedule the first timer tick. + UpdateTimer(); StartProcessingEdges(); }); @@ -93,7 +93,8 @@ ValueTask IActivationRebalancerSystemTarget.SetActivationCountOffset(int activat return ValueTask.CompletedTask; } - private void RegisterOrUpdateTimer(TimeSpan dueTime) + private void UpdateTimer() => UpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime)); + private void UpdateTimer(TimeSpan dueTime) { _timer.Change(dueTime, dueTime); LogPeriodicallyInvokeProtocol(_options.RebalancingPeriod, dueTime); @@ -101,6 +102,9 @@ private void RegisterOrUpdateTimer(TimeSpan dueTime) public async ValueTask TriggerExchangeRequest() { + // Schedule the next timer tick. + UpdateTimer(); + if (_currentExchangeSilo is not null) { // Skip this round if we are already in the process of exchanging with another silo. @@ -178,7 +182,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha // We pick some random time between 'min' and 'max' and than subtract from it 'min'. We do this so this silo doesn't have to wait for 'min + random', // as it did the very first time this was started. It is guaranteed that 'random - min' >= 0; as 'random' will be at the least equal to 'min'. - RegisterOrUpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime) - _options.MinRebalancingDueTime); + UpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime) - _options.MinRebalancingDueTime); LogMutualExchangeAttempt(request.SendingSilo); return AcceptExchangeResponse.CachedMutualExchangeAttempt; @@ -353,6 +357,7 @@ bool TryMigrateCore(MaxHeap sourceHeap, int localDel finally { _currentExchangeSilo = null; + UpdateTimer(); } } From 6457d691be3ea55bd60a3708a8e372e6a1e0f7bd Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Wed, 22 May 2024 08:11:37 -0700 Subject: [PATCH 11/28] minor clean up --- .../Options/ActiveRebalancingOptions.cs | 2 +- .../Rebalancing/ActivationRebalancer.Log.cs | 26 +++++++++++---- .../Rebalancing/ActivationRebalancer.cs | 33 ++++--------------- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index 9491249554..2c481bcf5d 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -25,7 +25,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public const uint DEFAULT_MAX_EDGE_COUNT = 10_000; + public const uint DEFAULT_MAX_EDGE_COUNT = 100_000; /// /// The minimum time given to this silo to gather statistics before triggering the first rebalancing cycle. diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs index e3ecf4ba5a..7079dc068b 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs @@ -20,21 +20,33 @@ internal partial class ActivationRebalancer [LoggerMessage(Level = LogLevel.Debug, Message = "I got an exchange request from {SendingSilo}, but I have been recently involved in another exchange {LastExchangeDuration} ago. My recovery period is {RecoveryPeriod}")] private partial void LogExchangedRecently(SiloAddress sendingSilo, TimeSpan lastExchangeDuration, TimeSpan recoveryPeriod); - [LoggerMessage(Level = LogLevel.Debug, Message = "Exchange request from {ThisSilo} failed, due to {CandidateSilo} having been recently involved in another exchange. I will try the next best candidate (if one is available), otherwise I will wait for my next period to come.")] + [LoggerMessage(Level = LogLevel.Debug, Message = "Exchange request from {ThisSilo} rejected: {CandidateSilo} was recently involved in another exchange. Attempting the next best candidate (if one is available) or waiting for my next period to come.")] private partial void LogExchangedRecentlyResponse(SiloAddress thisSilo, SiloAddress candidateSilo); - [LoggerMessage(Level = LogLevel.Debug, Message = "I got an exchange request from {SendingSilo}, but I am performing one with it at the same time. I have phase-shifted my timer to avoid these conflicts.")] + [LoggerMessage(Level = LogLevel.Debug, Message = "Rejecting exchange request from {SendingSilo} since we are already exchanging with that host.")] private partial void LogMutualExchangeAttempt(SiloAddress sendingSilo); [LoggerMessage(Level = LogLevel.Debug, Message = "Exchange request from {ThisSilo} superseded by a mutual exchange attempt with {CandidateSilo}.")] private partial void LogMutualExchangeAttemptResponse(SiloAddress thisSilo, SiloAddress candidateSilo); - [LoggerMessage(Level = LogLevel.Debug, Message = "I have successfully finalized my part of the exchange protocol. It was decided that I will give {GivingActivationCount} activations and take on a total of {TakingActivationCount} activations.")] + [LoggerMessage(Level = LogLevel.Debug, Message = "Finalized exchange protocol: migrating {GivingActivationCount} activations, receiving {TakingActivationCount} activations.")] private partial void LogProtocolFinalized(int givingActivationCount, int takingActivationCount); - [LoggerMessage(Level = LogLevel.Warning, Message = "An error occurred while performing exchange request from {ThisSilo} to {CandidateSilo}. I will try the next best candidate (if one is available), otherwise I will wait for my next period to come.")] + [LoggerMessage(Level = LogLevel.Warning, Message = "Error performing exchange request from {ThisSilo} to {CandidateSilo}. I will try the next best candidate (if one is available), otherwise I will wait for my next period to come.")] private partial void LogErrorOnProtocolExecution(Exception exception, SiloAddress thisSilo, SiloAddress candidateSilo); - [LoggerMessage(Level = LogLevel.Warning, Message = "There was an issue during the migration of the activation set initiated by {ThisSilo}.")] - private partial void LogErrorOnMigratingActivations(Exception exception, SiloAddress thisSilo); -} \ No newline at end of file + [LoggerMessage(Level = LogLevel.Warning, Message = "Error migrating exchange set.")] + private partial void LogErrorOnMigratingActivations(Exception exception); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Received AcceptExchangeRequest from {SendingSilo}, offering to send {ExchangeSetCount} activations from a total of {ActivationCount} activations.")] + private partial void LogReceivedExchangeRequest(SiloAddress sendingSilo, int exchangeSetCount, int activationCount); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Imbalance is {Imbalance} (remote: {RemoteCount} vs local {LocalCount})")] + private partial void LogImbalance(int imbalance, int remoteCount, int localCount); + + [LoggerMessage(Level = LogLevel.Debug, Message = "Computing transfer set took {Elapsed}. Anticipated imbalance after transfer is {AnticipatedImbalance}.")] + private partial void LogTransferSetComputed(TimeSpan elapsed, int anticipatedImbalance); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Error accepting exchange request from {SendingSilo}.")] + private partial void LogErrorAcceptingExchangeRequest(Exception exception, SiloAddress sendingSilo); +} diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 15434d5d46..0bffc039da 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -138,7 +138,6 @@ public async ValueTask TriggerExchangeRequest() LogBeginningProtocol(Silo, candidateSilo); var remoteRef = IActivationRebalancerSystemTarget.GetReference(_grainFactory, candidateSilo); - var sw2 = ValueStopwatch.StartNew(); var response = await remoteRef.AcceptExchangeRequest(new(Silo, [.. offeredGrains], GetLocalActivationCount())); switch (response.Type) @@ -174,7 +173,7 @@ public async ValueTask TriggerExchangeRequest() public async ValueTask AcceptExchangeRequest(AcceptExchangeRequest request) { - _logger.LogInformation("Received AcceptExchangeRequest from {Silo}", request.SendingSilo); + LogReceivedExchangeRequest(request.SendingSilo, request.ExchangeSet.Length, request.ActivationCountSnapshot); if (request.SendingSilo.Equals(_currentExchangeSilo) && Silo.CompareTo(request.SendingSilo) <= 0) { // Reject the request, as we are already in the process of exchanging with the sending silo. @@ -204,9 +203,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha var acceptedGrains = ImmutableArray.CreateBuilder(); var givingGrains = ImmutableArray.CreateBuilder(); var remoteSet = request.ExchangeSet; - _logger.LogInformation("About to create candidate set"); var localSet = GetCandidatesForSilo(GetMigrationCandidates(), request.SendingSilo); - _logger.LogInformation("Created candidate set"); // We need to determine 2 subsets: // - One that originates from sending silo (request.ExchangeSet) and will be (partially) accepted from this silo. @@ -216,12 +213,11 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha var initialImbalance = CalculateImbalance(remoteActivations, localActivations); int imbalance = initialImbalance; - _logger.LogInformation("Imbalance is {Imbalance} (remote: {RemoteCount} vs local {LocalCount})", imbalance, remoteActivations, localActivations); + LogImbalance(imbalance, remoteActivations, localActivations); var (localHeap, remoteHeap) = CreateCandidateHeaps(localSet, remoteSet); - _logger.LogInformation("Computing transfer set"); - var swTxs = ValueStopwatch.StartNew(); + var stopwatch = ValueStopwatch.StartNew(); var iterations = 0; while (true) { @@ -249,12 +245,10 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha break; } - _logger.LogInformation("Computing transfer set took {Elapsed}. Anticipated imbalance after transfer is {AnticipatedImbalance}", swTxs.Elapsed, imbalance); - swTxs.Restart(); + LogTransferSetComputed(stopwatch.Elapsed, imbalance); var giving = givingGrains.ToImmutable(); var accepting = acceptedGrains.ToImmutable(); await FinalizeProtocol(giving, accepting, request.SendingSilo); - _logger.LogInformation("Finalizing protocol based on provided set took {Elapsed}", swTxs.Elapsed); return new(AcceptExchangeResponse.ResponseType.Success, accepting, giving); @@ -351,7 +345,7 @@ bool TryMigrateCore(MaxHeap sourceHeap, int localDel } catch (Exception exception) { - _logger.LogError(exception, "Error accepting exchange request."); + LogErrorAcceptingExchangeRequest(exception, request.SendingSilo); throw; } finally @@ -436,13 +430,10 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr RequestContext.Set(IPlacementDirector.PlacementHintKey, targetSilo); List migrationTasks = []; - var sw1 = ValueStopwatch.StartNew(); - _logger.LogInformation("Telling {Count} grains to migrate from {LocalSilo} to {TargetSilo}", giving.Length, Silo, targetSilo); foreach (var grainId in giving) { migrationTasks.Add(_grainFactory.GetGrain(grainId).Cast().MigrateOnIdle().AsTask()); } - _logger.LogInformation("Telling {Count} grains to migrate took {Elapsed}", giving.Length, sw1.Elapsed); try { @@ -451,15 +442,12 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr catch (Exception) { // This should happen rarely, but at this point we cant really do much, as its out of our control. - // Even if some fail, at the end the algorithm will run again and eventually succeed with moving all activations were they belong. + // Even if some fail, the algorithm will eventually run again, so activations will have more chances to migrate. var aggEx = new AggregateException(migrationTasks.Select(t => t.Exception).Where(ex => ex is not null)!); - LogErrorOnMigratingActivations(aggEx, Silo); + LogErrorOnMigratingActivations(aggEx); } - _logger.LogInformation("Waiting for {Count} grains to migrate took {Elapsed}", giving.Length, sw1.Elapsed); - // Avoid mutating the source while enumerating it. - var sw = ValueStopwatch.StartNew(); var iterations = 0; var toRemove = new List(); @@ -498,8 +486,6 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr } } - _logger.LogInformation("Removing transfer set from edge weights took {Elapsed}.", sw.Elapsed); - // Stamp this silos exchange for a potential next pair exchange request. _lastExchangedStopwatch.Restart(); LogProtocolFinalized(giving.Length, accepting.Length); @@ -508,11 +494,8 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr private List<(SiloAddress Silo, List Candidates, long TransferScore)> CreateCandidateSets(ImmutableArray silos) { List<(SiloAddress Silo, List Candidates, long TransferScore)> candidateSets = new(silos.Length - 1); - var sw = ValueStopwatch.StartNew(); var localCandidates = GetMigrationCandidates(); - _logger.LogInformation("Computing local vertex edges took {Elapsed}.", sw.Elapsed); - sw.Restart(); foreach (var siloAddress in silos) { if (siloAddress.Equals(Silo)) @@ -527,8 +510,6 @@ private List<(SiloAddress Silo, List Candidates, long TransferS candidateSets.Add(new(siloAddress, [.. candidatesForRemote], totalAccTransferScore)); } - _logger.LogInformation("Computing candidate set per-silo took {Elapsed}.", sw.Elapsed); - // Order them by the highest accumulated transfer score candidateSets.Sort(static (a, b) => -a.TransferScore.CompareTo(b.TransferScore)); From 5c65e8ea44d69dae8171e4432cc449a5a55e2c62 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Wed, 22 May 2024 21:37:58 -0700 Subject: [PATCH 12/28] Ignore remote vertex instance if it appears in the local set. --- .../Rebalancing/ActivationRebalancer.cs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 0bffc039da..488a62396d 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -17,6 +17,7 @@ using Orleans.Runtime.Utilities; using System.Runtime.InteropServices; using System.Diagnostics.CodeAnalysis; +using Microsoft.CodeAnalysis; namespace Orleans.Runtime.Placement.Rebalancing; @@ -369,11 +370,11 @@ private static (MaxHeap Local, MaxHeap index = []; + Dictionary heapIndex = []; List localVertexList = new(local.Count); foreach (var element in local) { - var vertex = CreateVertex(sourceIndex, index, element); + var vertex = CreateVertex(sourceIndex, heapIndex, element); vertex.Location = VertexLocation.Local; localVertexList.Add(vertex); } @@ -381,7 +382,13 @@ private static (MaxHeap Local, MaxHeap remoteVertexList = new(remote.Length); foreach (var element in remote) { - var vertex = CreateVertex(sourceIndex, index, element); + var vertex = CreateVertex(sourceIndex, heapIndex, element); + if (vertex.Location is not VertexLocation.Unknown) + { + // This vertex is already part of the local set, so assume that the vertex is local and ignore the remote vertex. + continue; + } + vertex.Location = VertexLocation.Remote; remoteVertexList.Add(vertex); } @@ -439,7 +446,7 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr { await Task.WhenAll(migrationTasks); } - catch (Exception) + catch { // This should happen rarely, but at this point we cant really do much, as its out of our control. // Even if some fail, the algorithm will eventually run again, so activations will have more chances to migrate. @@ -694,4 +701,4 @@ private enum Direction : byte /// The edge's direction /// The number of estimated messages exchanged between and . private readonly record struct VertexEdge(GrainId SourceId, GrainId TargetId, bool IsMigratable, SiloAddress PartnerSilo, Direction Direction, long Weight); -} \ No newline at end of file +} From 2fb64303635daafd105772b567e947fe17000895 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Thu, 23 May 2024 07:43:23 -0700 Subject: [PATCH 13/28] Improve timer scheduling slightly - consider winding back into UpdateTimer --- .../Options/ActiveRebalancingOptions.cs | 2 +- .../Rebalancing/ActivationRebalancer.cs | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index 2c481bcf5d..9491249554 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -25,7 +25,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public const uint DEFAULT_MAX_EDGE_COUNT = 100_000; + public const uint DEFAULT_MAX_EDGE_COUNT = 10_000; /// /// The minimum time given to this silo to gather statistics before triggering the first rebalancing cycle. diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 488a62396d..b0b547ed03 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -30,6 +30,7 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe private readonly IRebalancingMessageFilter _messageFilter; private readonly IImbalanceToleranceRule _toleranceRule; private readonly ActivationDirectory _activationDirectory; + private readonly TimeProvider _timeProvider; private readonly ActiveRebalancingOptions _options; private readonly StripedMpscBuffer _pendingMessages; private readonly SingleWaiterAutoResetEvent _pendingMessageEvent = new() { RunContinuationsAsynchronously = true }; @@ -49,7 +50,8 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe IImbalanceToleranceRule toleranceRule, ActivationDirectory activationDirectory, Catalog catalog, - IOptions options) + IOptions options, + TimeProvider timeProvider) : base(Constants.ActivationRebalancerType, localSiloDetails.SiloAddress, loggerFactory) { _logger = loggerFactory.CreateLogger(); @@ -59,10 +61,11 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe _messageFilter = messageFilter; _toleranceRule = toleranceRule; _activationDirectory = activationDirectory; + _timeProvider = timeProvider; _edgeWeights = new((int)options.Value.MaxEdgeCount); _pendingMessages = new StripedMpscBuffer(Environment.ProcessorCount, options.Value.MaxUnprocessedEdges / Environment.ProcessorCount); - _lastExchangedStopwatch = CoarseStopwatch.StartNew((long)options.Value.RecoveryPeriod.Add(TimeSpan.FromDays(2)).TotalMilliseconds); + _lastExchangedStopwatch = CoarseStopwatch.StartNew(); catalog.RegisterSystemTarget(this); _siloStatusOracle.SubscribeToSiloStatusEvents(this); _timer = RegisterTimer(_ => TriggerExchangeRequest().AsTask(), null, Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); @@ -103,6 +106,13 @@ private void UpdateTimer(TimeSpan dueTime) public async ValueTask TriggerExchangeRequest() { + var coolDown = _options.RebalancingPeriod - _lastExchangedStopwatch.Elapsed; + if (coolDown > TimeSpan.Zero) + { + _logger.LogDebug("Waiting an additional {CoolDown} to cool down before initiating the exchange protocol.", coolDown); + await Task.Delay(coolDown, _timeProvider); + } + // Schedule the next timer tick. UpdateTimer(); @@ -352,7 +362,6 @@ bool TryMigrateCore(MaxHeap sourceHeap, int localDel finally { _currentExchangeSilo = null; - UpdateTimer(); } } @@ -555,7 +564,7 @@ private List GetCandidatesForSilo(List Date: Fri, 24 May 2024 17:20:48 -0700 Subject: [PATCH 14/28] Remove an option, fix tests --- .../Options/ActiveRebalancingOptions.cs | 72 +++++++------------ .../Rebalancing/ActivationRebalancer.Log.cs | 4 +- .../Rebalancing/ActivationRebalancer.cs | 17 +++-- .../CustomToleranceTests.cs | 4 +- .../DefaultToleranceTests.cs | 4 +- .../ActiveRebalancingTests/OptionsTests.cs | 35 +++++---- 6 files changed, 61 insertions(+), 75 deletions(-) diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index 9491249554..4ee5a3ccbc 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -20,60 +20,42 @@ public sealed class ActiveRebalancingOptions /// In order to preserve memory, the most heaviest links are recorded in a probabilistic way, so there is an inherent error associated with that. /// That error is inversely proportional to this value, so values under 100 are not recommended. If you notice that the system is not converging fast enough, do consider increasing this number. /// - public uint MaxEdgeCount { get; set; } = DEFAULT_MAX_EDGE_COUNT; + public int MaxEdgeCount { get; set; } = DEFAULT_MAX_EDGE_COUNT; /// /// The default value of . /// - public const uint DEFAULT_MAX_EDGE_COUNT = 10_000; + public const int DEFAULT_MAX_EDGE_COUNT = 10_000; /// - /// The minimum time given to this silo to gather statistics before triggering the first rebalancing cycle. + /// The minimum time between initiating a rebalancing cycle. /// - /// The actual due time is picked randomly between this and . - public TimeSpan MinRebalancingDueTime { get; set; } = DEFAULT_MINUMUM_REBALANCING_DUE_TIME; + /// The actual due time is picked randomly between this and . + public TimeSpan MinRebalancingPeriod { get; set; } = DEFAULT_MINUMUM_REBALANCING_PERIOD; /// - /// The default value of . + /// The default value of . /// - public static readonly TimeSpan DEFAULT_MINUMUM_REBALANCING_DUE_TIME = TimeSpan.FromMinutes(1); + public static readonly TimeSpan DEFAULT_MINUMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(1); /// - /// The maximum time given to this silo to gather statistics before triggering the first rebalancing cycle. + /// The maximum time between initiating a rebalancing cycle. /// /// - /// The actual due time is picked randomly between this and . - /// For optimal results, you should aim to give this an extra 10 seconds x the maximum anticipated silo count in the cluster. + /// The actual due time is picked randomly between this and . + /// For optimal results, you should aim to give this an extra 10 seconds multiplied by the maximum anticipated silo count in the cluster. /// - public TimeSpan MaxRebalancingDueTime { get; set; } = DEFAULT_MAXIMUM_REBALANCING_DUE_TIME; + public TimeSpan MaxRebalancingPeriod { get; set; } = DEFAULT_MAXIMUM_REBALANCING_PERIOD; /// - /// The default value of . + /// The default value of . /// - public static readonly TimeSpan DEFAULT_MAXIMUM_REBALANCING_DUE_TIME = TimeSpan.FromMinutes(2); - - /// - /// The cycle upon which this silo will trigger a rebalancing session with another silo. - /// - /// Must be greater than , you should aim for at least 2 times that of . - public TimeSpan RebalancingPeriod { get; set; } = DEFAULT_REBALANCING_PERIOD; - - /// - /// The default value of . - /// - public static readonly TimeSpan DEFAULT_REBALANCING_PERIOD = TimeSpan.FromMinutes(2); + public static readonly TimeSpan DEFAULT_MAXIMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(2); /// /// The minimum time needed for a silo to recover from a previous rebalancing. /// Until this time has elapsed, this silo will not take part in any rebalancing attempt from another silo. /// - /// - /// - /// While this silo will refuse rebalancing attempts from other silos, if falls within this period, than - /// this silo will attempt a rebalancing with another silo, but this silo will be the initiator, not the other way around. - /// - /// Must be less than , you should aim for at least 1/2 times that of . - /// public TimeSpan RecoveryPeriod { get; set; } = DEFAULT_RECOVERY_PERIOD; /// @@ -98,45 +80,45 @@ internal sealed class ActiveRebalancingOptionsValidator(IOptions throw new OrleansConfigurationException($"{propertyName} must be greater than 0"); - private static void ThrowMustBeGreaterThan(string name1, string name2) - => throw new OrleansConfigurationException($"{name1} must be greater than {name2}"); + private static void ThrowMustBeGreaterThanOrEqualTo(string name1, string name2) + => throw new OrleansConfigurationException($"{name1} must be greater than or equal to {name2}"); } \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs index 7079dc068b..4b42c00152 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.Log.cs @@ -5,8 +5,8 @@ namespace Orleans.Runtime.Placement.Rebalancing; internal partial class ActivationRebalancer { - [LoggerMessage(Level = LogLevel.Debug, Message = "I will periodically initiate the exchange protocol every {RebalancingPeriod} starting in {DueTime}.")] - private partial void LogPeriodicallyInvokeProtocol(TimeSpan rebalancingPeriod, TimeSpan dueTime); + [LoggerMessage(Level = LogLevel.Debug, Message = "I will periodically initiate the exchange protocol every {MinRebalancingPeriod} to {MaxRebalancingPeriod} starting in {DueTime}.")] + private partial void LogPeriodicallyInvokeProtocol(TimeSpan minRebalancingPeriod, TimeSpan maxRebalancingPeriod, TimeSpan dueTime); [LoggerMessage(Level = LogLevel.Debug, Message = "Active rebalancing is enabled, but the cluster contains only one silo. Waiting for at least another silo to join the cluster to proceed.")] private partial void LogSingleSiloCluster(); diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index b0b547ed03..80465ad6e6 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -97,16 +97,16 @@ ValueTask IActivationRebalancerSystemTarget.SetActivationCountOffset(int activat return ValueTask.CompletedTask; } - private void UpdateTimer() => UpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime)); + private void UpdateTimer() => UpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingPeriod, _options.MaxRebalancingPeriod)); private void UpdateTimer(TimeSpan dueTime) { _timer.Change(dueTime, dueTime); - LogPeriodicallyInvokeProtocol(_options.RebalancingPeriod, dueTime); + LogPeriodicallyInvokeProtocol(_options.MinRebalancingPeriod, _options.MaxRebalancingPeriod, dueTime); } public async ValueTask TriggerExchangeRequest() { - var coolDown = _options.RebalancingPeriod - _lastExchangedStopwatch.Elapsed; + var coolDown = _options.RecoveryPeriod - _lastExchangedStopwatch.Elapsed; if (coolDown > TimeSpan.Zero) { _logger.LogDebug("Waiting an additional {CoolDown} to cool down before initiating the exchange protocol.", coolDown); @@ -133,7 +133,10 @@ public async ValueTask TriggerExchangeRequest() return; } - foreach ((var candidateSilo, var offeredGrains, var _) in CreateCandidateSets(silos)) + var sw = ValueStopwatch.StartNew(); + var sets = CreateCandidateSets(silos); + _logger.LogInformation("Candidate sets computed in {Elapsed} ms.", sw.Elapsed.TotalMilliseconds); + foreach ((var candidateSilo, var offeredGrains, var _) in sets) { if (offeredGrains.Count == 0) { @@ -192,7 +195,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha // We pick some random time between 'min' and 'max' and than subtract from it 'min'. We do this so this silo doesn't have to wait for 'min + random', // as it did the very first time this was started. It is guaranteed that 'random - min' >= 0; as 'random' will be at the least equal to 'min'. - UpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingDueTime, _options.MaxRebalancingDueTime) - _options.MinRebalancingDueTime); + UpdateTimer(RandomTimeSpan.Next(_options.MinRebalancingPeriod, _options.MaxRebalancingPeriod) - _options.MinRebalancingPeriod); LogMutualExchangeAttempt(request.SendingSilo); return AcceptExchangeResponse.CachedMutualExchangeAttempt; @@ -226,9 +229,11 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha int imbalance = initialImbalance; LogImbalance(imbalance, remoteActivations, localActivations); + var stopwatch = ValueStopwatch.StartNew(); var (localHeap, remoteHeap) = CreateCandidateHeaps(localSet, remoteSet); + _logger.LogInformation("Candidate heaps created in {Elapsed} ms.", stopwatch.Elapsed.TotalMilliseconds); + stopwatch.Restart(); - var stopwatch = ValueStopwatch.StartNew(); var iterations = 0; while (true) { diff --git a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs index b1f7f08d61..20b1917ace 100644 --- a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs @@ -206,8 +206,8 @@ public void Configure(ISiloBuilder hostBuilder) .Configure(o => { // Make these so that the timers practically never fire! We will invoke the protocol manually. - o.MinRebalancingDueTime = TimeSpan.FromSeconds(299); - o.MaxRebalancingDueTime = TimeSpan.FromSeconds(300); + o.MinRebalancingPeriod = TimeSpan.FromSeconds(299); + o.MaxRebalancingPeriod = TimeSpan.FromSeconds(300); // Make this practically zero, so we can invoke the protocol more than once without needing to put a delay in the tests. o.RecoveryPeriod = TimeSpan.FromMilliseconds(1); }) diff --git a/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs index 9795f0a6c4..59cc082799 100644 --- a/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs @@ -601,8 +601,8 @@ public void Configure(ISiloBuilder hostBuilder) .Configure(o => { // Make these so that the timers practically never fire! We will invoke the protocol manually. - o.MinRebalancingDueTime = TimeSpan.FromSeconds(299); - o.MaxRebalancingDueTime = TimeSpan.FromSeconds(300); + o.MinRebalancingPeriod = TimeSpan.FromSeconds(299); + o.MaxRebalancingPeriod = TimeSpan.FromSeconds(300); // Make this practically zero, so we can invoke the protocol more than once without needing to put a delay in the tests. o.RecoveryPeriod = TimeSpan.FromMilliseconds(1); }) diff --git a/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs b/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs index 6ca3830421..e35e928d64 100644 --- a/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs @@ -11,35 +11,34 @@ public class OptionsTests [Fact] public void ConstantsShouldNotChange() { - Assert.Equal(10_000u, ActiveRebalancingOptions.DEFAULT_MAX_EDGE_COUNT); - Assert.Equal(TimeSpan.FromMinutes(1), ActiveRebalancingOptions.DEFAULT_MINUMUM_REBALANCING_DUE_TIME); - Assert.Equal(TimeSpan.FromMinutes(2), ActiveRebalancingOptions.DEFAULT_MAXIMUM_REBALANCING_DUE_TIME); - Assert.Equal(TimeSpan.FromMinutes(2), ActiveRebalancingOptions.DEFAULT_REBALANCING_PERIOD); + Assert.Equal(10_000, ActiveRebalancingOptions.DEFAULT_MAX_EDGE_COUNT); + Assert.Equal(TimeSpan.FromMinutes(1), ActiveRebalancingOptions.DEFAULT_MINUMUM_REBALANCING_PERIOD); + Assert.Equal(TimeSpan.FromMinutes(2), ActiveRebalancingOptions.DEFAULT_MAXIMUM_REBALANCING_PERIOD); Assert.Equal(TimeSpan.FromMinutes(1), ActiveRebalancingOptions.DEFAULT_RECOVERY_PERIOD); } [Theory] - [InlineData(0, 1, 1, 2, 1)] - [InlineData(1, 0, 1, 2, 1)] - [InlineData(1, 1, 0, 2, 1)] + [InlineData(0, 1, 1, 1, 1)] + [InlineData(1, 0, 1, 1, 1)] + [InlineData(1, 1, 0, 1, 1)] [InlineData(1, 1, 1, 0, 1)] - [InlineData(1, 1, 1, 2, 0)] - [InlineData(1, 2, 1, 2, 1)] - [InlineData(1, 2, 1, 1, 2)] + [InlineData(1, 1, 1, 1, 0)] + [InlineData(1, 1, 2, 1, 1)] + [InlineData(1, 1, 2, 1, 2)] public void InvalidOptionsShouldThrow( - uint topHeaviestCommunicationLinks, - int minimumRebalancingDueTimeMinutes, - int maximumRebalancingDueTimeMinutes, - int rebalancingPeriodMinutes, + int topHeaviestCommunicationLinks, + int maxUnprocessedEdges, + int minRebalancingPeriodMinutes, + int maxRebalancingPeriodMinutes, int recoveryPeriodMinutes) { var options = new ActiveRebalancingOptions { MaxEdgeCount = topHeaviestCommunicationLinks, - MinRebalancingDueTime = TimeSpan.FromMinutes(minimumRebalancingDueTimeMinutes), - MaxRebalancingDueTime = TimeSpan.FromMinutes(maximumRebalancingDueTimeMinutes), - RebalancingPeriod = TimeSpan.FromMinutes(rebalancingPeriodMinutes), - RecoveryPeriod = TimeSpan.FromMinutes(recoveryPeriodMinutes) + MinRebalancingPeriod = TimeSpan.FromMinutes(minRebalancingPeriodMinutes), + MaxRebalancingPeriod = TimeSpan.FromMinutes(maxRebalancingPeriodMinutes), + RecoveryPeriod = TimeSpan.FromMinutes(recoveryPeriodMinutes), + MaxUnprocessedEdges = maxUnprocessedEdges }; var validator = new ActiveRebalancingOptionsValidator(Options.Create(options)); From f158228df088041b9b13778a0b00645035cfe271 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Sun, 26 May 2024 10:46:51 -0700 Subject: [PATCH 15/28] Fix message sink cycles per yield --- .../Rebalancing/ActivationRebalancer.MessageSink.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index c8e5c8ee8f..b44ae64208 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -42,11 +42,11 @@ public async Task StopProcessingEdgesAsync(CancellationToken cancellationToken) private async Task ProcessPendingEdges(CancellationToken cancellationToken) { - const int MaxCyclesPerYield = 100; + const int MaxIterationsPerYield = 100; await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding | ConfigureAwaitOptions.ContinueOnCapturedContext); var drainBuffer = new Message[256]; - var cyclesPerYield = 100; + var iteration = 0; while (!cancellationToken.IsCancellationRequested) { var count = _pendingMessages.DrainTo(drainBuffer); @@ -67,9 +67,9 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) await _pendingMessageEvent.WaitAsync(); } - if (++cyclesPerYield >= MaxCyclesPerYield) + if (++iteration >= MaxIterationsPerYield) { - cyclesPerYield = 0; + iteration = 0; await Task.Yield(); } } From 5600ca1f38a557ef2aa2a50a15a0833c1e4692a4 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Sun, 26 May 2024 11:49:20 -0700 Subject: [PATCH 16/28] Experiment: anchoring grains with globally negative transfer score --- .../ActivationRebalancer.MessageSink.cs | 31 +++++++++- .../Rebalancing/ActivationRebalancer.cs | 60 ++++++++++++++++--- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index b44ae64208..5c2ca6e1c7 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -1,4 +1,5 @@ #nullable enable +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -40,6 +41,7 @@ public async Task StopProcessingEdgesAsync(CancellationToken cancellationToken) } } + private readonly HashSet _anchoredGrainIds = new(); private async Task ProcessPendingEdges(CancellationToken cancellationToken) { const int MaxIterationsPerYield = 100; @@ -56,9 +58,32 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) { _messageFilter.IsAcceptable(message, out var isSenderMigratable, out var isTargetMigratable); - Edge edge = new( - new(message.SendingGrain, message.SendingSilo, isSenderMigratable), - new(message.TargetGrain, message.TargetSilo, isTargetMigratable)); + EdgeVertex sourceVertex, destinationVertex; + if (_anchoredGrainIds.Contains(message.SendingGrain)) + { + sourceVertex = new(GrainId, Silo, isMigratable: false); + } + else + { + sourceVertex = new(message.SendingGrain, message.SendingSilo, isSenderMigratable); + } + + if (_anchoredGrainIds.Contains(message.TargetGrain)) + { + destinationVertex = new(GrainId, Silo, isMigratable: false); + } + else + { + destinationVertex = new(message.TargetGrain, message.TargetSilo, isTargetMigratable); + } + + if (!isSenderMigratable && !isTargetMigratable) + { + // Ignore edges between two non-migratable grains. + continue; + } + + Edge edge = new(sourceVertex, destinationVertex); _edgeWeights.Add(edge); } } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 80465ad6e6..89c06ca526 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -134,7 +134,9 @@ public async ValueTask TriggerExchangeRequest() } var sw = ValueStopwatch.StartNew(); - var sets = CreateCandidateSets(silos); + var migrationCandidates = GetMigrationCandidates(); + var sets = CreateCandidateSets(migrationCandidates, silos); + var anchoredSet = ComputeAnchoredGrains(migrationCandidates); _logger.LogInformation("Candidate sets computed in {Elapsed} ms.", sw.Elapsed.TotalMilliseconds); foreach ((var candidateSilo, var offeredGrains, var _) in sets) { @@ -158,7 +160,7 @@ public async ValueTask TriggerExchangeRequest() { case AcceptExchangeResponse.ResponseType.Success: // Exchange was successful, no need to iterate over another candidate. - await FinalizeProtocol(response.AcceptedGrainIds, response.GivenGrainIds, candidateSilo); + await FinalizeProtocol(response.AcceptedGrainIds, response.GivenGrainIds, candidateSilo, anchoredSet); return; case AcceptExchangeResponse.ResponseType.ExchangedRecently: // The remote silo has been recently involved in another exchange, try the next best candidate. @@ -217,7 +219,9 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha var acceptedGrains = ImmutableArray.CreateBuilder(); var givingGrains = ImmutableArray.CreateBuilder(); var remoteSet = request.ExchangeSet; - var localSet = GetCandidatesForSilo(GetMigrationCandidates(), request.SendingSilo); + var migrationCandidates = GetMigrationCandidates(); + var localSet = GetCandidatesForSilo(migrationCandidates, request.SendingSilo); + var anchoredSet = ComputeAnchoredGrains(migrationCandidates); // We need to determine 2 subsets: // - One that originates from sending silo (request.ExchangeSet) and will be (partially) accepted from this silo. @@ -264,7 +268,7 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha LogTransferSetComputed(stopwatch.Elapsed, imbalance); var giving = givingGrains.ToImmutable(); var accepting = acceptedGrains.ToImmutable(); - await FinalizeProtocol(giving, accepting, request.SendingSilo); + await FinalizeProtocol(giving, accepting, request.SendingSilo, anchoredSet); return new(AcceptExchangeResponse.ResponseType.Success, accepting, giving); @@ -445,7 +449,7 @@ static CandidateVertexHeapElement GetOrAddVertex(Dictionary /// The grain ids to migrate to the remote host. /// The grain ids to which are migrating to the local host. - private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArray accepting, SiloAddress targetSilo) + private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArray accepting, SiloAddress targetSilo, HashSet newlyAnchoredGrains) { // The protocol concluded that 'this' silo should take on 'set', so we hint to the director accordingly. RequestContext.Set(IPlacementDirector.PlacementHintKey, targetSilo); @@ -473,6 +477,17 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr var toRemove = new List(); var affected = new HashSet(giving.Length + accepting.Length); + _logger.LogInformation("Adding {NewlyAnchoredGrains} newly anchored grains to set of {AllAnchoredGrainsCount} on host {Silo}. EdgeWeights contains {EdgeWeightCount} elements.", newlyAnchoredGrains.Count, _anchoredGrainIds.Count, Silo, _edgeWeights.Count); + foreach (var id in newlyAnchoredGrains) + { + _anchoredGrainIds.Add(id); + } + + foreach (var id in _anchoredGrainIds) + { + affected.Add(id); + } + foreach (var id in accepting) { affected.Add(id); @@ -512,10 +527,9 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr LogProtocolFinalized(giving.Length, accepting.Length); } - private List<(SiloAddress Silo, List Candidates, long TransferScore)> CreateCandidateSets(ImmutableArray silos) + private List<(SiloAddress Silo, List Candidates, long TransferScore)> CreateCandidateSets(List> migrationCandidates, ImmutableArray silos) { List<(SiloAddress Silo, List Candidates, long TransferScore)> candidateSets = new(silos.Length - 1); - var localCandidates = GetMigrationCandidates(); foreach (var siloAddress in silos) { @@ -525,7 +539,7 @@ private List<(SiloAddress Silo, List Candidates, long TransferS continue; } - var candidatesForRemote = GetCandidatesForSilo(localCandidates, siloAddress); + var candidatesForRemote = GetCandidatesForSilo(migrationCandidates, siloAddress); var totalAccTransferScore = candidatesForRemote.Sum(x => x.AccumulatedTransferScore); candidateSets.Add(new(siloAddress, [.. candidatesForRemote], totalAccTransferScore)); @@ -596,6 +610,36 @@ private List GetCandidatesForSilo(List ComputeAnchoredGrains(List> migrationCandidates) + { + HashSet anchoredGrains = []; + foreach (var grainEdges in migrationCandidates) + { + var accLocalScore = 0L; + var accRemoteScore = 0L; + + foreach (var edge in grainEdges) + { + if (edge.Direction is Direction.LocalToLocal) + { + accLocalScore += edge.Weight; + } + else + { + Debug.Assert(edge.Direction is Direction.RemoteToLocal or Direction.LocalToRemote); + accRemoteScore += edge.Weight; + } + } + + if (accLocalScore > accRemoteScore) + { + anchoredGrains.Add(grainEdges.Key); + } + } + + return anchoredGrains; + } + private List> GetMigrationCandidates() => CreateLocalVertexEdges().Where(x => x.IsMigratable).GroupBy(x => x.SourceId).ToList(); /// From 22183f178ba03e94d47453980ca795d750f6cd1f Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 27 May 2024 11:56:57 -0700 Subject: [PATCH 17/28] Add initial bloom filter implementation --- .../Placement/Rebalancing/BloomFilter.cs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs diff --git a/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs new file mode 100644 index 0000000000..9f384a9bac --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs @@ -0,0 +1,68 @@ +using System; +using System.Diagnostics; +using System.IO.Hashing; +using System.Linq; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace Orleans.Runtime.Placement.Rebalancing; + +internal sealed class BloomFilter +{ + private const double Ln2Squared = 0.4804530139182014246671025263266649717305529515945455; + private const double Ln2 = 0.6931471805599453094172321214581765680755001343602552; + private readonly int[] _hashFuncSeeds; + private readonly int[] _filter; + private readonly int _indexMask; + + public BloomFilter(int capacity, double falsePositiveRate) + { + // Calculate the ideal bloom filter size and hash code count for the given (estimated) capacity and desired false positive rate. + // See https://en.wikipedia.org/wiki/Bloom_filter. + var minBitCount = (int)(-1 / Ln2Squared * capacity * Math.Log(falsePositiveRate)) / 8; + var arraySize = (int)CeilingPowerOfTwo((uint)(minBitCount - 1 + (1 << 5)) >> 5); + _indexMask = arraySize - 1; + _filter = new int[arraySize]; + + var hashFuncCount = (int)Math.Min(minBitCount * 8 / capacity * Ln2 / 2, 8); + Debug.Assert(hashFuncCount > 0); + _hashFuncSeeds = Enumerable.Range(0, hashFuncCount).Select(p => (int)unchecked(p * 0xFBA4C795 + 1)).ToArray(); + Debug.Assert(_hashFuncSeeds.Length == hashFuncCount); + } + + public void Add(GrainId id) + { + foreach (var seed in _hashFuncSeeds) + { + var indexes = XxHash3.HashToUInt64(id.Key.AsSpan(), (long)seed << 32 | id.GetUniformHashCode()); + Set((int)indexes); + Set((int)(indexes >> 32)); + } + } + + public bool Contains(GrainId id) + { + foreach (var seed in _hashFuncSeeds) + { + var indexes = XxHash3.HashToUInt64(id.Key.AsSpan(), (long)seed << 32 | id.GetUniformHashCode()); + var clear = IsClear((int)indexes); + clear |= IsClear((int)(indexes >> 32)); + if (clear) + { + return false; + } + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsClear(int index) => (_filter[(index >> 5) & _indexMask] & (1 << index)) == 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Set(int index) => _filter[(index >> 5) & _indexMask] |= 1 << index; + + public void Reset() => Array.Clear(_filter); + + private static uint CeilingPowerOfTwo(uint x) => 1u << -BitOperations.LeadingZeroCount(x - 1); +} From eaf36ee41a317d7f82e4e7c504af17118f1de81b Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 27 May 2024 12:15:54 -0700 Subject: [PATCH 18/28] Add & use bloom filter to filter 'anchored' grains from Top-K --- .../ActivationRebalancer.MessageSink.cs | 30 +- .../Rebalancing/ActivationRebalancer.cs | 9 +- .../Rebalancing/FrequentItemCollection.cs | 10 +- .../Rebalancing/RebalancingMessageFilter.cs | 34 +- test/Benchmarks/TopK/BloomFilterBenchmark.cs | 196 ++++++++++ test/Benchmarks/TopK/TopKBenchmark.cs | 352 +++++++++--------- .../BloomFilterTests.cs | 24 ++ .../FrequencyFilterTests.cs | 2 +- .../TestMessageFilter.cs | 1 - 9 files changed, 428 insertions(+), 230 deletions(-) create mode 100644 test/Benchmarks/TopK/BloomFilterBenchmark.cs create mode 100644 test/TesterInternal/ActiveRebalancingTests/BloomFilterTests.cs diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index 5c2ca6e1c7..3d258cd84e 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -1,5 +1,4 @@ #nullable enable -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -11,6 +10,10 @@ namespace Orleans.Runtime.Placement.Rebalancing; internal partial class ActivationRebalancer : IMessageStatisticsSink { private readonly CancellationTokenSource _shutdownCts = new(); + + // This bloom filter contains grain ids which will are anchored to the current silo. + // Ids are inserted when a grain is found to have a negative transfer score. + private readonly BloomFilter _anchoredGrainIds = new(100_000, 0.01); private Task? _processPendingEdgesTask; public void StartProcessingEdges() @@ -41,7 +44,6 @@ public async Task StopProcessingEdgesAsync(CancellationToken cancellationToken) } } - private readonly HashSet _anchoredGrainIds = new(); private async Task ProcessPendingEdges(CancellationToken cancellationToken) { const int MaxIterationsPerYield = 100; @@ -56,9 +58,12 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) { foreach (var message in drainBuffer[..count]) { - _messageFilter.IsAcceptable(message, out var isSenderMigratable, out var isTargetMigratable); + if (!_messageFilter.IsAcceptable(message, out var isSenderMigratable, out var isTargetMigratable)) + { + continue; + } - EdgeVertex sourceVertex, destinationVertex; + EdgeVertex sourceVertex; if (_anchoredGrainIds.Contains(message.SendingGrain)) { sourceVertex = new(GrainId, Silo, isMigratable: false); @@ -68,6 +73,7 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) sourceVertex = new(message.SendingGrain, message.SendingSilo, isSenderMigratable); } + EdgeVertex destinationVertex; if (_anchoredGrainIds.Contains(message.TargetGrain)) { destinationVertex = new(GrainId, Silo, isMigratable: false); @@ -77,7 +83,7 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) destinationVertex = new(message.TargetGrain, message.TargetSilo, isTargetMigratable); } - if (!isSenderMigratable && !isTargetMigratable) + if (!sourceVertex.IsMigratable && !destinationVertex.IsMigratable) { // Ignore edges between two non-migratable grains. continue; @@ -102,7 +108,19 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) public void RecordMessage(Message message) { - if (!_enableMessageSampling || !_messageFilter.IsAcceptable(message, out var isSenderMigratable, out var isTargetMigratable)) + if (!_enableMessageSampling || message.IsSystemMessage) + { + return; + } + + // It must have a direction, and must not be a 'response' as it would skew analysis. + if (message.HasDirection is false || message.Direction == Message.Directions.Response) + { + return; + } + + // Sender and target need to be fully addressable to know where to move to or towards. + if (!message.IsSenderFullyAddressed || !message.IsTargetFullyAddressed) { return; } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 89c06ca526..c5566e96ec 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -477,17 +477,12 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr var toRemove = new List(); var affected = new HashSet(giving.Length + accepting.Length); - _logger.LogInformation("Adding {NewlyAnchoredGrains} newly anchored grains to set of {AllAnchoredGrainsCount} on host {Silo}. EdgeWeights contains {EdgeWeightCount} elements.", newlyAnchoredGrains.Count, _anchoredGrainIds.Count, Silo, _edgeWeights.Count); + _logger.LogInformation("Adding {NewlyAnchoredGrains} newly anchored grains to set on host {Silo}. EdgeWeights contains {EdgeWeightCount} elements.", newlyAnchoredGrains.Count, Silo, _edgeWeights.Count); foreach (var id in newlyAnchoredGrains) { _anchoredGrainIds.Add(id); } - foreach (var id in _anchoredGrainIds) - { - affected.Add(id); - } - foreach (var id in accepting) { affected.Add(id); @@ -502,7 +497,7 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr { foreach (var (edge, count, error) in _edgeWeights.Elements) { - if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id)) + if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id) || _anchoredGrainIds.Contains(edge.Source.Id) || _anchoredGrainIds.Contains(edge.Target.Id)) { toRemove.Add(edge); } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs b/src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs index 00b0846754..ed1354b81e 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/FrequentItemCollection.cs @@ -133,6 +133,11 @@ public void Add(in TElement element) /// if matching entry was found and removed, otherwise. protected bool RemoveCore(TKey key) { + // Remove the element from the sketch + var sketchMask = _sketch.Length - 1; + var sketchHash = key.GetHashCode(); + _sketch[sketchHash & sketchMask] = 0; + // Remove the element from the heap index if (!_heapIndex.Remove(key, out var index)) { @@ -153,11 +158,6 @@ protected bool RemoveCore(TKey key) nodes[newSize] = default; - // Remove the element from the sketch - var sketchMask = _sketch.Length - 1; - var sketchHash = key.GetHashCode(); - _sketch[sketchHash & sketchMask] = 0; - return true; } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs index 82127cb3c0..031b741512 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/RebalancingMessageFilter.cs @@ -4,7 +4,6 @@ using System.Collections.Concurrent; using System.Collections.Frozen; using System.Runtime.CompilerServices; -using System.Threading; namespace Orleans.Runtime.Placement.Rebalancing; @@ -30,24 +29,6 @@ public bool IsAcceptable(Message message, out bool isSenderMigratable, out bool isSenderMigratable = false; isTargetMigratable = false; - // Ignore system messages - if (message.IsSystemMessage) - { - return false; - } - - // It must have a direction, and must not be a 'response' as it would skew analysis. - if (message.HasDirection is false || message.Direction == Message.Directions.Response) - { - return false; - } - - // Sender and target need to be fully addressable to know where to move to or towards. - if (!message.IsSenderFullyAddressed || !message.IsTargetFullyAddressed) - { - return false; - } - // There are some edge cases when this can happen i.e. a grain invoking another one of its methods via AsReference<>, but we still exclude it // as wherever this grain would be located in the cluster, it would always be a local call (since it targets itself), this would add negative transfer cost // which would skew a potential relocation of this grain, while it shouldn't, because whenever this grain is located, it would still make local calls to itself. @@ -56,24 +37,11 @@ public bool IsAcceptable(Message message, out bool isSenderMigratable, out bool return false; } - // Ignore rebalancer messages: either to another rebalancer, or when executing migration requests to activations. - if (IsRebalancer(message.SendingGrain.Type) || IsRebalancer(message.TargetGrain.Type)) - { - return false; - } - isSenderMigratable = IsMigratable(message.SendingGrain.Type); isTargetMigratable = IsMigratable(message.TargetGrain.Type); // If both are not migratable types we ignore this. But if one of them is not, then we allow passing, as we wish to move grains closer to them, as with any type of grain. - if (!isSenderMigratable && !isTargetMigratable) - { - return false; - } - - return true; - - bool IsRebalancer(GrainType grainType) => grainType.Equals(Constants.ActivationRebalancerType); + return isSenderMigratable || isTargetMigratable; bool IsMigratable(GrainType grainType) { diff --git a/test/Benchmarks/TopK/BloomFilterBenchmark.cs b/test/Benchmarks/TopK/BloomFilterBenchmark.cs new file mode 100644 index 0000000000..4b999d2c52 --- /dev/null +++ b/test/Benchmarks/TopK/BloomFilterBenchmark.cs @@ -0,0 +1,196 @@ +using System.Collections; +using System.IO.Hashing; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Reports; +using Benchmarks.Utilities; +using Orleans.Runtime.Placement.Rebalancing; + +namespace Benchmarks.TopK; + +[MemoryDiagnoser] +[FalsePositiveRateColumn] +public class BloomFilterBenchmark +{ + private BloomFilter _bloomFilter; + private BloomFilter _bloomFilterWithSamples; + private OriginalBloomFilter _originalBloomFilter; + private OriginalBloomFilter _originalBloomFilterWithSamples; + private GrainId[] _population; + private HashSet _set; + private ZipfRejectionSampler _sampler; + private GrainId[] _samples; + + [Params(1_000_000, Priority = 3)] + public int Pop { get; set; } + + [Params(/*0.2, 0.4, 0.6, 0.8, */1.02 /*, 1.2, 1.4, 1.6*/, Priority = 2)] + public double Skew { get; set; } + + [Params(1_000_000, Priority = 1)] + public int Cap { get; set; } + + [Params(10_000, Priority = 4)] + public int Samples { get; set; } + + [GlobalSetup] + public void GlobalSetup() + { + _population = new GrainId[Pop]; + _sampler = new(new Random(42), Pop, Skew); + for (var i = 0; i < Pop; i++) + { + _population[i] = GrainId.Create($"grain_{i}", i.ToString()); + } + + _bloomFilter = new(Cap, 0.01); + _bloomFilterWithSamples = new(Cap, 0.01); + _originalBloomFilter = new(); + _originalBloomFilterWithSamples = new(); + + _samples = new GrainId[Samples]; + _set = new(Samples); + for (var i = 0; i < Samples; i++) + { + //var sample = _sampler.Sample(); + var value = _population[i]; + _samples[i] = value; + _set.Add(value); + _bloomFilterWithSamples.Add(value); + _originalBloomFilterWithSamples.Add(value); + } + } + + [Benchmark] + [BenchmarkCategory("Add")] + public void BloomFilter_Add() + { + foreach (var sample in _samples) + { + _bloomFilter.Add(sample); + } + } + + [Benchmark] + [BenchmarkCategory("Contains")] + public void BloomFilter_Contains() + { + foreach (var sample in _samples) + { + _bloomFilterWithSamples.Contains(sample); + } + } + + /* + [Benchmark] + [BenchmarkCategory("FP rate")] + public int BloomFilter_FPR() + { + var correct = 0; + var incorrect = 0; + foreach (var sample in _population) + { + if (!_bloomFilterWithSamples.Contains(sample) == _set.Contains(sample)) + { + correct++; + } + else + { + incorrect++; + } + } + + return incorrect; + } + */ + + [Benchmark] + [BenchmarkCategory("Add")] + public void OriginalBloomFilter_Add() + { + foreach (var sample in _samples) + { + _originalBloomFilter.Add(sample); + } + } + + [Benchmark] + [BenchmarkCategory("Contains")] + public void OriginalBloomFilter_Contains() + { + foreach (var sample in _samples) + { + _originalBloomFilterWithSamples.Contains(sample); + } + } + + /* + [Benchmark] + [BenchmarkCategory("FP rate")] + public int OriginalBloomFilter_FPR() + { + var correct = 0; + var incorrect = 0; + foreach (var sample in _population) + { + if (!_originalBloomFilterWithSamples.Contains(sample) == _set.Contains(sample)) + { + correct++; + } + else + { + incorrect++; + } + } + + return incorrect; + } + */ +} + +[AttributeUsage(AttributeTargets.Class)] +public class FalsePositiveRateColumnAttribute : Attribute, IConfigSource +{ + public FalsePositiveRateColumnAttribute(string columnName = "FP %") + { + var config = ManualConfig.CreateEmpty(); + config.AddColumn( + new MethodResultColumn(columnName, + val => + { + return $"{val}"; + })); + Config = config; + } + + public IConfig Config { get; } +} +public class OriginalBloomFilter +{ + private const int bitArraySize = 1_198_132; // formula 8 * n / ln(2) -> for 1000 elements, 0.01% + private readonly int[] hashFuncSeeds = Enumerable.Range(0, 6).Select(p => (int)unchecked(p * 0xFBA4C795 + 1)).ToArray(); + private readonly BitArray filterBits = new(bitArraySize); + + public void Add(GrainId id) + { + foreach (int s in hashFuncSeeds) + { + uint i = XxHash32.HashToUInt32(id.Key.AsSpan(), s); + filterBits.Set((int)(i % (uint)filterBits.Length), true); + } + } + + public bool Contains(GrainId id) + { + foreach (int s in hashFuncSeeds) + { + uint i = XxHash32.HashToUInt32(id.Key.AsSpan(), s); + if (!filterBits.Get((int)(i % (uint)filterBits.Length))) + { + return false; + } + } + return true; + } +} diff --git a/test/Benchmarks/TopK/TopKBenchmark.cs b/test/Benchmarks/TopK/TopKBenchmark.cs index 8f6af29783..5e0b197616 100644 --- a/test/Benchmarks/TopK/TopKBenchmark.cs +++ b/test/Benchmarks/TopK/TopKBenchmark.cs @@ -3,7 +3,6 @@ using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using Orleans.Placement.Rebalancing; -using Orleans.Runtime; using Orleans.Runtime.Placement.Rebalancing; namespace Benchmarks.TopK; @@ -155,45 +154,6 @@ private sealed class UlongFrequentItemCollection(int capacity) : FrequentItemCol public void Clear() => ClearCore(); } - // https://jasoncrease.medium.com/rejection-sampling-the-zipf-distribution-6b359792cffa - public class ZipfRejectionSampler - { - private readonly Random _rand; - private readonly double _skew; - private readonly double _t; - - public ZipfRejectionSampler(Random random, long cardinality, double skew) - { - _rand = random; - _skew = skew; - _t = (Math.Pow(cardinality, 1 - skew) - skew) / (1 - skew); - } - - public long Sample() - { - while (true) - { - double invB = bInvCdf(_rand.NextDouble()); - long sampleX = (long)(invB + 1); - double yRand = _rand.NextDouble(); - double ratioTop = Math.Pow(sampleX, -_skew); - double ratioBottom = sampleX <= 1 ? 1 / _t : Math.Pow(invB, -_skew) / _t; - double rat = (ratioTop) / (ratioBottom * _t); - - if (yRand < rat) - return sampleX; - } - } - private double bInvCdf(double p) - { - if (p * _t <= 1) - return p * _t; - else - return Math.Pow((p * _t) * (1 - _skew) + _skew, 1 / (1 - _skew)); - } - } - - internal class EdgeCounter(ulong value, Edge edge) { public ulong Value { get; set; } = value; @@ -201,191 +161,229 @@ internal class EdgeCounter(ulong value, Edge edge) } -/// -/// Implementation of the Space-Saving algorithm: https://www.cse.ust.hk/~raywong/comp5331/References/EfficientComputationOfFrequentAndTop-kElementsInDataStreams.pdf -/// -internal sealed class FrequencySink(int capacity) -{ - public ulong GetKey(in Edge element) => (ulong)element.Source.Id.GetUniformHashCode() << 32 | element.Target.Id.GetUniformHashCode(); - private readonly Dictionary _counters = new(capacity); - private readonly UpdateableMinHeap _heap = new(capacity); - - public int Capacity { get; } = capacity; - public Dictionary.ValueCollection Counters => _counters.Values; - - public void Add(Edge edge) + /// + /// Implementation of the Space-Saving algorithm: https://www.cse.ust.hk/~raywong/comp5331/References/EfficientComputationOfFrequentAndTop-kElementsInDataStreams.pdf + /// + internal sealed class FrequencySink(int capacity) { - var combinedHash = GetKey(edge); - if (_counters.TryGetValue(combinedHash, out var counter)) - { - counter.Value++; - _heap.Update(combinedHash, counter.Value); + public ulong GetKey(in Edge element) => (ulong)element.Source.Id.GetUniformHashCode() << 32 | element.Target.Id.GetUniformHashCode(); + private readonly Dictionary _counters = new(capacity); + private readonly UpdateableMinHeap _heap = new(capacity); - return; - } + public int Capacity { get; } = capacity; + public Dictionary.ValueCollection Counters => _counters.Values; - if (_counters.Count == Capacity) + public void Add(Edge edge) { - var minHash = _heap.Dequeue(); - _counters.Remove(minHash); - } + var combinedHash = GetKey(edge); + if (_counters.TryGetValue(combinedHash, out var counter)) + { + counter.Value++; + _heap.Update(combinedHash, counter.Value); - _counters.Add(combinedHash, new EdgeCounter(1, edge)); - _heap.Enqueue(combinedHash, _counters[combinedHash].Value); - } + return; + } - public void Remove(uint sourceHash, uint targetHash) - { - var combinedHash = CombineHashes(sourceHash, targetHash); - var reversedHash = CombineHashes(targetHash, sourceHash); + if (_counters.Count == Capacity) + { + var minHash = _heap.Dequeue(); + _counters.Remove(minHash); + } - if (_counters.Remove(combinedHash)) - { - _ = _heap.Remove(combinedHash); + _counters.Add(combinedHash, new EdgeCounter(1, edge)); + _heap.Enqueue(combinedHash, _counters[combinedHash].Value); } - if (_counters.Remove(reversedHash)) + public void Remove(uint sourceHash, uint targetHash) { - _ = _heap.Remove(reversedHash); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ulong CombineHashes(uint sourceHash, uint targetHash) - => (ulong)sourceHash << 32 | targetHash; + var combinedHash = CombineHashes(sourceHash, targetHash); + var reversedHash = CombineHashes(targetHash, sourceHash); - // Inspired by: https://github.com/DesignEngrLab/TVGL/blob/master/TessellationAndVoxelizationGeometryLibrary/Miscellaneous%20Functions/UpdatablePriorityQueue.cs - private class UpdateableMinHeap(int capacity) - { - private const int Arity = 4; - private const int Log2Arity = 2; + if (_counters.Remove(combinedHash)) + { + _ = _heap.Remove(combinedHash); + } - private readonly Dictionary _hashIndexes = new(capacity); - private readonly (ulong Hash, ulong Value)[] _nodes = new (ulong, ulong)[capacity]; + if (_counters.Remove(reversedHash)) + { + _ = _heap.Remove(reversedHash); + } + } - private int _size; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong CombineHashes(uint sourceHash, uint targetHash) + => (ulong)sourceHash << 32 | targetHash; - public void Enqueue(ulong hash, ulong value) + // Inspired by: https://github.com/DesignEngrLab/TVGL/blob/master/TessellationAndVoxelizationGeometryLibrary/Miscellaneous%20Functions/UpdatablePriorityQueue.cs + private class UpdateableMinHeap(int capacity) { - var currentSize = _size; - _size = currentSize + 1; + private const int Arity = 4; + private const int Log2Arity = 2; - MoveNodeUp((hash, value), currentSize); - } + private readonly Dictionary _hashIndexes = new(capacity); + private readonly (ulong Hash, ulong Value)[] _nodes = new (ulong, ulong)[capacity]; - public ulong Dequeue() - { - var hash = _nodes[0].Hash; - _hashIndexes.Remove(hash); + private int _size; - var lastNodeIndex = --_size; - if (lastNodeIndex > 0) + public void Enqueue(ulong hash, ulong value) { - var lastNode = _nodes[lastNodeIndex]; - MoveNodeDown(lastNode, 0); - } + var currentSize = _size; + _size = currentSize + 1; - return hash; - } + MoveNodeUp((hash, value), currentSize); + } - public bool Remove(ulong hash) - { - if (!_hashIndexes.TryGetValue(hash, out var index)) + public ulong Dequeue() { - return false; - } + var hash = _nodes[0].Hash; + _hashIndexes.Remove(hash); - var nodes = _nodes; - var newSize = --_size; + var lastNodeIndex = --_size; + if (lastNodeIndex > 0) + { + var lastNode = _nodes[lastNodeIndex]; + MoveNodeDown(lastNode, 0); + } - if (index < newSize) - { - var lastNode = nodes[newSize]; - MoveNodeDown(lastNode, index); + return hash; } - _hashIndexes.Remove(hash); - nodes[newSize] = default; + public bool Remove(ulong hash) + { + if (!_hashIndexes.TryGetValue(hash, out var index)) + { + return false; + } - return true; - } + var nodes = _nodes; + var newSize = --_size; - public void Update(ulong hash, ulong newValue) - { - Remove(hash); - Enqueue(hash, newValue); - } + if (index < newSize) + { + var lastNode = nodes[newSize]; + MoveNodeDown(lastNode, index); + } - private void MoveNodeUp((ulong Hash, ulong Value) node, int nodeIndex) - { - Debug.Assert(0 <= nodeIndex && nodeIndex < _size); + _hashIndexes.Remove(hash); + nodes[newSize] = default; - var nodes = _nodes; + return true; + } - while (nodeIndex > 0) + public void Update(ulong hash, ulong newValue) { - var parentIndex = GetParentIndex(nodeIndex); - var parent = nodes[parentIndex]; - - if (Comparer.Default.Compare(node.Value, parent.Value) < 0) - { - nodes[nodeIndex] = parent; - _hashIndexes[parent.Hash] = nodeIndex; - nodeIndex = parentIndex; - } - else - { - break; - } + Remove(hash); + Enqueue(hash, newValue); } - _hashIndexes[node.Hash] = nodeIndex; - nodes[nodeIndex] = node; + private void MoveNodeUp((ulong Hash, ulong Value) node, int nodeIndex) + { + Debug.Assert(0 <= nodeIndex && nodeIndex < _size); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static int GetParentIndex(int index) => (index - 1) >> Log2Arity; - } + var nodes = _nodes; - private void MoveNodeDown((ulong Hash, ulong Value) node, int nodeIndex) - { - Debug.Assert(0 <= nodeIndex && nodeIndex < _size); + while (nodeIndex > 0) + { + var parentIndex = GetParentIndex(nodeIndex); + var parent = nodes[parentIndex]; + + if (Comparer.Default.Compare(node.Value, parent.Value) < 0) + { + nodes[nodeIndex] = parent; + _hashIndexes[parent.Hash] = nodeIndex; + nodeIndex = parentIndex; + } + else + { + break; + } + } - var nodes = _nodes; - var size = _size; + _hashIndexes[node.Hash] = nodeIndex; + nodes[nodeIndex] = node; - int i; - while ((i = GetFirstChildIndex(nodeIndex)) < size) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int GetParentIndex(int index) => (index - 1) >> Log2Arity; + } + + private void MoveNodeDown((ulong Hash, ulong Value) node, int nodeIndex) { - var minChild = nodes[i]; - var minChildIndex = i; + Debug.Assert(0 <= nodeIndex && nodeIndex < _size); - var childIndexUpperBound = Math.Min(i + Arity, size); - while (++i < childIndexUpperBound) + var nodes = _nodes; + var size = _size; + + int i; + while ((i = GetFirstChildIndex(nodeIndex)) < size) { - var nextChild = nodes[i]; - if (nextChild.Value < minChild.Value) + var minChild = nodes[i]; + var minChildIndex = i; + + var childIndexUpperBound = Math.Min(i + Arity, size); + while (++i < childIndexUpperBound) { - minChild = nextChild; - minChildIndex = i; + var nextChild = nodes[i]; + if (nextChild.Value < minChild.Value) + { + minChild = nextChild; + minChildIndex = i; + } } - } - if (node.Value <= minChild.Value) - { - break; + if (node.Value <= minChild.Value) + { + break; + } + + nodes[nodeIndex] = minChild; + _hashIndexes[minChild.Hash] = nodeIndex; + nodeIndex = minChildIndex; } - nodes[nodeIndex] = minChild; - _hashIndexes[minChild.Hash] = nodeIndex; - nodeIndex = minChildIndex; + _hashIndexes[node.Hash] = nodeIndex; + nodes[nodeIndex] = node; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int GetFirstChildIndex(int index) => (index << Log2Arity) + 1; } + } + } +} - _hashIndexes[node.Hash] = nodeIndex; - nodes[nodeIndex] = node; + // https://jasoncrease.medium.com/rejection-sampling-the-zipf-distribution-6b359792cffa + internal sealed class ZipfRejectionSampler + { + private readonly Random _rand; + private readonly double _skew; + private readonly double _t; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static int GetFirstChildIndex(int index) => (index << Log2Arity) + 1; + public ZipfRejectionSampler(Random random, long cardinality, double skew) + { + _rand = random; + _skew = skew; + _t = (Math.Pow(cardinality, 1 - skew) - skew) / (1 - skew); + } + + public long Sample() + { + while (true) + { + double invB = bInvCdf(_rand.NextDouble()); + long sampleX = (long)(invB + 1); + double yRand = _rand.NextDouble(); + double ratioTop = Math.Pow(sampleX, -_skew); + double ratioBottom = sampleX <= 1 ? 1 / _t : Math.Pow(invB, -_skew) / _t; + double rat = (ratioTop) / (ratioBottom * _t); + + if (yRand < rat) + return sampleX; + } + } + private double bInvCdf(double p) + { + if (p * _t <= 1) + return p * _t; + else + return Math.Pow((p * _t) * (1 - _skew) + _skew, 1 / (1 - _skew)); } } -} -} diff --git a/test/TesterInternal/ActiveRebalancingTests/BloomFilterTests.cs b/test/TesterInternal/ActiveRebalancingTests/BloomFilterTests.cs new file mode 100644 index 0000000000..20f0fc5a27 --- /dev/null +++ b/test/TesterInternal/ActiveRebalancingTests/BloomFilterTests.cs @@ -0,0 +1,24 @@ +using Orleans.Runtime.Placement.Rebalancing; +using Xunit; + +namespace UnitTests.ActiveRebalancingTests; + +public class BloomFilterTests +{ + [Fact] + public void AddAndCheck() + { + var bloomFilter = new BloomFilter(100, 0.01); + var sample = new GrainId(GrainType.Create("type"), IdSpan.Create("key")); + bloomFilter.Add(sample); + Assert.True(bloomFilter.Contains(sample)); + } + + [Fact] + public void DoesNotContainSome() + { + var bloomFilter = new BloomFilter(100, 0.01); + var sample = new GrainId(GrainType.Create("type"), IdSpan.Create("key")); + Assert.False(bloomFilter.Contains(sample)); + } +} diff --git a/test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs b/test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs index 8e9a59c508..5089c6368e 100644 --- a/test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/FrequencyFilterTests.cs @@ -1,4 +1,4 @@ -using System.Text; +using System.Text; using Orleans.Runtime.Placement.Rebalancing; using Xunit; diff --git a/test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs b/test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs index d5a9070da0..b9b1735416 100644 --- a/test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs +++ b/test/TesterInternal/ActiveRebalancingTests/TestMessageFilter.cs @@ -1,4 +1,3 @@ -using Orleans.Runtime; using Orleans.Runtime.Placement; using Orleans.Runtime.Placement.Rebalancing; From 390f2af3a6b220a9cf73e36a767476aa843d7c13 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 27 May 2024 14:49:28 -0700 Subject: [PATCH 19/28] Mix instead of re-hashing. Yield only every 25ms+ --- .../Rebalancing/ActivationRebalancer.cs | 8 +++-- .../Placement/Rebalancing/BloomFilter.cs | 34 ++++++++++++++----- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index c5566e96ec..a163f69ecf 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -239,11 +239,13 @@ public async ValueTask AcceptExchangeRequest(AcceptExcha stopwatch.Restart(); var iterations = 0; + var yieldStopwatch = CoarseStopwatch.StartNew(); while (true) { - if (++iterations % 128 == 0) + if (++iterations % 128 == 0 && yieldStopwatch.ElapsedMilliseconds > 25) { // Give other tasks a chance to execute periodically. + yieldStopwatch.Restart(); await Task.Delay(1); } @@ -493,6 +495,7 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr affected.Add(id); } + var yieldStopwatch = CoarseStopwatch.StartNew(); if (affected.Count > 0) { foreach (var (edge, count, error) in _edgeWeights.Elements) @@ -505,9 +508,10 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr foreach (var edge in toRemove) { - if (++iterations % 128 == 0) + if (++iterations % 128 == 0 && yieldStopwatch.ElapsedMilliseconds > 25) { // Give other tasks a chance to execute periodically. + yieldStopwatch.Restart(); await Task.Delay(1); } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs index 9f384a9bac..f28e29e388 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs @@ -11,7 +11,7 @@ internal sealed class BloomFilter { private const double Ln2Squared = 0.4804530139182014246671025263266649717305529515945455; private const double Ln2 = 0.6931471805599453094172321214581765680755001343602552; - private readonly int[] _hashFuncSeeds; + private readonly ulong[] _hashFuncSeeds; private readonly int[] _filter; private readonly int _indexMask; @@ -24,29 +24,32 @@ public BloomFilter(int capacity, double falsePositiveRate) _indexMask = arraySize - 1; _filter = new int[arraySize]; + // Divide the hash count by 2 since we are using 64-bit hash codes split into two 32-bit hash codes. var hashFuncCount = (int)Math.Min(minBitCount * 8 / capacity * Ln2 / 2, 8); Debug.Assert(hashFuncCount > 0); - _hashFuncSeeds = Enumerable.Range(0, hashFuncCount).Select(p => (int)unchecked(p * 0xFBA4C795 + 1)).ToArray(); + _hashFuncSeeds = Enumerable.Range(0, hashFuncCount).Select(p => (ulong)unchecked(p * 0xFBA4C795 + 1)).ToArray(); Debug.Assert(_hashFuncSeeds.Length == hashFuncCount); } public void Add(GrainId id) { + var hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); foreach (var seed in _hashFuncSeeds) { - var indexes = XxHash3.HashToUInt64(id.Key.AsSpan(), (long)seed << 32 | id.GetUniformHashCode()); - Set((int)indexes); - Set((int)(indexes >> 32)); + hash = Mix64(hash ^ seed); + Set((int)hash); + Set((int)(hash >> 32)); } } public bool Contains(GrainId id) { + var hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); foreach (var seed in _hashFuncSeeds) { - var indexes = XxHash3.HashToUInt64(id.Key.AsSpan(), (long)seed << 32 | id.GetUniformHashCode()); - var clear = IsClear((int)indexes); - clear |= IsClear((int)(indexes >> 32)); + hash = Mix64(hash ^ seed); + var clear = IsClear((int)hash); + clear |= IsClear((int)(hash >> 32)); if (clear) { return false; @@ -62,6 +65,21 @@ public bool Contains(GrainId id) [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Set(int index) => _filter[(index >> 5) & _indexMask] |= 1 << index; + /// + /// Computes Stafford variant 13 of 64-bit mix function. + /// + /// The input parameter. + /// A bit mix of the input parameter. + /// + /// See http://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html + /// + public static ulong Mix64(ulong z) + { + z = (z ^ z >> 30) * 0xbf58476d1ce4e5b9L; + z = (z ^ z >> 27) * 0x94d049bb133111ebL; + return z ^ z >> 31; + } + public void Reset() => Array.Clear(_filter); private static uint CeilingPowerOfTwo(uint x) => 1u << -BitOperations.LeadingZeroCount(x - 1); From 20bb8ca1db5fa91d1fda146e5a8f3c2cf557501e Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 27 May 2024 15:57:51 -0700 Subject: [PATCH 20/28] Improvements --- .../ActivationRebalancer.MessageSink.cs | 15 +++++---------- .../Placement/Rebalancing/BloomFilter.cs | 2 +- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index 3d258cd84e..f97b2576fa 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -1,4 +1,5 @@ #nullable enable +using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -46,11 +47,9 @@ public async Task StopProcessingEdgesAsync(CancellationToken cancellationToken) private async Task ProcessPendingEdges(CancellationToken cancellationToken) { - const int MaxIterationsPerYield = 100; await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding | ConfigureAwaitOptions.ContinueOnCapturedContext); - var drainBuffer = new Message[256]; - var iteration = 0; + var drainBuffer = new Message[128]; while (!cancellationToken.IsCancellationRequested) { var count = _pendingMessages.DrainTo(drainBuffer); @@ -92,17 +91,13 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) Edge edge = new(sourceVertex, destinationVertex); _edgeWeights.Add(edge); } + + await Task.Delay(TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond), CancellationToken.None); } else { await _pendingMessageEvent.WaitAsync(); } - - if (++iteration >= MaxIterationsPerYield) - { - iteration = 0; - await Task.Yield(); - } } } @@ -114,7 +109,7 @@ public void RecordMessage(Message message) } // It must have a direction, and must not be a 'response' as it would skew analysis. - if (message.HasDirection is false || message.Direction == Message.Directions.Response) + if (message.Direction is Message.Directions.None or Message.Directions.Response) { return; } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs index f28e29e388..947ca35a29 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs @@ -27,7 +27,7 @@ public BloomFilter(int capacity, double falsePositiveRate) // Divide the hash count by 2 since we are using 64-bit hash codes split into two 32-bit hash codes. var hashFuncCount = (int)Math.Min(minBitCount * 8 / capacity * Ln2 / 2, 8); Debug.Assert(hashFuncCount > 0); - _hashFuncSeeds = Enumerable.Range(0, hashFuncCount).Select(p => (ulong)unchecked(p * 0xFBA4C795 + 1)).ToArray(); + _hashFuncSeeds = Enumerable.Range(0, hashFuncCount).Select(p => unchecked((ulong)p * 0xFBA4C795FBA4C795 + 1)).ToArray(); Debug.Assert(_hashFuncSeeds.Length == hashFuncCount); } From 0bea27dc3805b01b5e77c872ca4b5b7df4c7b893 Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 27 May 2024 15:59:17 -0700 Subject: [PATCH 21/28] REVERT - short recovery --- .../Configuration/Options/ActiveRebalancingOptions.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index 4ee5a3ccbc..533b52054f 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -36,7 +36,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public static readonly TimeSpan DEFAULT_MINUMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(1); + public static readonly TimeSpan DEFAULT_MINUMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(0.5); /// /// The maximum time between initiating a rebalancing cycle. @@ -50,7 +50,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public static readonly TimeSpan DEFAULT_MAXIMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(2); + public static readonly TimeSpan DEFAULT_MAXIMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(1); /// /// The minimum time needed for a silo to recover from a previous rebalancing. @@ -61,7 +61,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public static readonly TimeSpan DEFAULT_RECOVERY_PERIOD = TimeSpan.FromMinutes(1); + public static readonly TimeSpan DEFAULT_RECOVERY_PERIOD = TimeSpan.FromMinutes(0.5); /// /// The maximum number of unprocessed edges to buffer. If this number is exceeded, the oldest edges will be discarded. From 4e79452ae4540528f1f3497603167787f059408b Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 27 May 2024 16:25:42 -0700 Subject: [PATCH 22/28] Yield more in MessageSink --- .../Rebalancing/ActivationRebalancer.MessageSink.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index f97b2576fa..ef8744327a 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -50,6 +50,8 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding | ConfigureAwaitOptions.ContinueOnCapturedContext); var drainBuffer = new Message[128]; + var iteration = 0; + const int MaxIterationsPerYield = 128; while (!cancellationToken.IsCancellationRequested) { var count = _pendingMessages.DrainTo(drainBuffer); @@ -92,10 +94,15 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) _edgeWeights.Add(edge); } - await Task.Delay(TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond), CancellationToken.None); + if (++iteration >= MaxIterationsPerYield) + { + iteration = 0; + await Task.Delay(TimeSpan.FromTicks(TimeSpan.TicksPerMillisecond), CancellationToken.None); + } } else { + iteration = 0; await _pendingMessageEvent.WaitAsync(); } } From e438cca68baad5f8347822ecb8c6b6333864d48a Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Mon, 27 May 2024 18:22:25 -0700 Subject: [PATCH 23/28] WIP Aspire playground for Active Rebalancing --- Directory.Packages.props | 5 + Orleans.sln | 26 + .../DashboardToy.Common/Class1.cs | 6 + .../DashboardToy.Common.csproj | 14 + .../DashboardToy.Frontend.csproj | 19 + .../Data/ClusterDiagnosticsService.cs | 141 +++++ .../DashboardToy.Frontend/Program.cs | 138 +++++ .../Properties/launchSettings.json | 37 ++ .../appsettings.Development.json | 9 + .../DashboardToy.Frontend/appsettings.json | 9 + .../wwwroot/css/bootstrap/bootstrap.min.css | 7 + .../css/bootstrap/bootstrap.min.css.map | 1 + .../wwwroot/css/open-iconic/FONT-LICENSE | 86 +++ .../wwwroot/css/open-iconic/ICON-LICENSE | 21 + .../wwwroot/css/open-iconic/README.md | 114 ++++ .../font/css/open-iconic-bootstrap.min.css | 1 + .../open-iconic/font/fonts/open-iconic.eot | Bin 0 -> 28196 bytes .../open-iconic/font/fonts/open-iconic.otf | Bin 0 -> 20996 bytes .../open-iconic/font/fonts/open-iconic.svg | 543 ++++++++++++++++++ .../open-iconic/font/fonts/open-iconic.ttf | Bin 0 -> 28028 bytes .../open-iconic/font/fonts/open-iconic.woff | Bin 0 -> 14984 bytes .../wwwroot/css/site.css | 68 +++ .../DashboardToy.Frontend/wwwroot/favicon.png | Bin 0 -> 1148 bytes .../DashboardToy.Frontend/wwwroot/index.html | 353 ++++++++++++ .../DashboardToy.AppHost.csproj | 22 + .../DashboradToy.AppHost/Program.cs | 13 + .../Properties/launchSettings.json | 29 + .../appsettings.Development.json | 8 + .../DashboradToy.AppHost/appsettings.json | 9 + .../IActivationRebalancerSystemTarget.cs | 5 + .../IManagementGrain.cs | 20 + src/Orleans.Runtime/Catalog/ActivationData.cs | 2 +- .../Catalog/ActivationDirectory.cs | 3 +- .../Options/ActiveRebalancingOptions.cs | 6 +- .../ResourceOptimizedPlacementOptions.cs | 2 +- src/Orleans.Runtime/Core/ManagementGrain.cs | 45 ++ .../ActivationRebalancer.MessageSink.cs | 4 +- .../Rebalancing/ActivationRebalancer.cs | 68 ++- .../Placement/Rebalancing/MaxHeap.cs | 1 + .../BenchmarkGrains}/Ping/TreeGrain.cs | 6 +- .../CustomToleranceTests.cs | 2 + .../ActiveRebalancingTests/MaxHeapTests.cs | 4 +- 42 files changed, 1815 insertions(+), 32 deletions(-) create mode 100644 playground/DashboardToy/DashboardToy.Common/Class1.cs create mode 100644 playground/DashboardToy/DashboardToy.Common/DashboardToy.Common.csproj create mode 100644 playground/DashboardToy/DashboardToy.Frontend/DashboardToy.Frontend.csproj create mode 100644 playground/DashboardToy/DashboardToy.Frontend/Data/ClusterDiagnosticsService.cs create mode 100644 playground/DashboardToy/DashboardToy.Frontend/Program.cs create mode 100644 playground/DashboardToy/DashboardToy.Frontend/Properties/launchSettings.json create mode 100644 playground/DashboardToy/DashboardToy.Frontend/appsettings.Development.json create mode 100644 playground/DashboardToy/DashboardToy.Frontend/appsettings.json create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css.map create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/FONT-LICENSE create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/ICON-LICENSE create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/README.md create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/font/css/open-iconic-bootstrap.min.css create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/font/fonts/open-iconic.eot create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/font/fonts/open-iconic.otf create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/font/fonts/open-iconic.svg create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/font/fonts/open-iconic.ttf create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/open-iconic/font/fonts/open-iconic.woff create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/site.css create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/favicon.png create mode 100644 playground/DashboardToy/DashboardToy.Frontend/wwwroot/index.html create mode 100644 playground/DashboardToy/DashboradToy.AppHost/DashboardToy.AppHost.csproj create mode 100644 playground/DashboardToy/DashboradToy.AppHost/Program.cs create mode 100644 playground/DashboardToy/DashboradToy.AppHost/Properties/launchSettings.json create mode 100644 playground/DashboardToy/DashboradToy.AppHost/appsettings.Development.json create mode 100644 playground/DashboardToy/DashboradToy.AppHost/appsettings.json rename test/{Benchmarks => Grains/BenchmarkGrains}/Ping/TreeGrain.cs (90%) diff --git a/Directory.Packages.props b/Directory.Packages.props index 34392f9573..4414bd45cf 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -50,6 +50,11 @@ + + + + + diff --git a/Orleans.sln b/Orleans.sln index 4df07f0c88..a8d210e770 100644 --- a/Orleans.sln +++ b/Orleans.sln @@ -223,6 +223,16 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Orleans.Streaming.AdoNet", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmarks.AdoNet", "test\Benchmarks.AdoNet\Benchmarks.AdoNet.csproj", "{B8F43537-2D2E-42A0-BE67-5E07E4313AEA}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "playground", "playground", "{A41DE3D1-F8AA-4234-BE6F-3C9646A1507A}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DashboardToy", "DashboardToy", "{316CDCC7-323F-4264-9FC9-667662BB1F80}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DashboardToy.Frontend", "playground\DashboardToy\DashboardToy.Frontend\DashboardToy.Frontend.csproj", "{C4DD4F96-3EC6-47C6-97AA-9B14F0F2099B}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DashboardToy.AppHost", "playground\DashboardToy\DashboradToy.AppHost\DashboardToy.AppHost.csproj", "{84B44F1D-B7FE-40E3-82F0-730A55AC8613}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DashboardToy.Common", "playground\DashboardToy\DashboardToy.Common\DashboardToy.Common.csproj", "{10F842A4-D5F9-41A7-B328-6D5A02BBE4C9}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -593,6 +603,18 @@ Global {B8F43537-2D2E-42A0-BE67-5E07E4313AEA}.Debug|Any CPU.Build.0 = Debug|Any CPU {B8F43537-2D2E-42A0-BE67-5E07E4313AEA}.Release|Any CPU.ActiveCfg = Release|Any CPU {B8F43537-2D2E-42A0-BE67-5E07E4313AEA}.Release|Any CPU.Build.0 = Release|Any CPU + {C4DD4F96-3EC6-47C6-97AA-9B14F0F2099B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4DD4F96-3EC6-47C6-97AA-9B14F0F2099B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4DD4F96-3EC6-47C6-97AA-9B14F0F2099B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4DD4F96-3EC6-47C6-97AA-9B14F0F2099B}.Release|Any CPU.Build.0 = Release|Any CPU + {84B44F1D-B7FE-40E3-82F0-730A55AC8613}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84B44F1D-B7FE-40E3-82F0-730A55AC8613}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84B44F1D-B7FE-40E3-82F0-730A55AC8613}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84B44F1D-B7FE-40E3-82F0-730A55AC8613}.Release|Any CPU.Build.0 = Release|Any CPU + {10F842A4-D5F9-41A7-B328-6D5A02BBE4C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10F842A4-D5F9-41A7-B328-6D5A02BBE4C9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10F842A4-D5F9-41A7-B328-6D5A02BBE4C9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10F842A4-D5F9-41A7-B328-6D5A02BBE4C9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -702,6 +724,10 @@ Global {A073C0EE-8732-42F9-A22E-D47034E25076} = {4CD3AA9E-D937-48CA-BB6C-158E12257D23} {2B994F33-16CF-4679-936A-5AEABC529D2C} = {EB2EDE59-5021-42EE-A97A-D59939B39C66} {B8F43537-2D2E-42A0-BE67-5E07E4313AEA} = {2CAB7894-777C-42B1-8B1E-322868CE92C7} + {316CDCC7-323F-4264-9FC9-667662BB1F80} = {A41DE3D1-F8AA-4234-BE6F-3C9646A1507A} + {C4DD4F96-3EC6-47C6-97AA-9B14F0F2099B} = {316CDCC7-323F-4264-9FC9-667662BB1F80} + {84B44F1D-B7FE-40E3-82F0-730A55AC8613} = {316CDCC7-323F-4264-9FC9-667662BB1F80} + {10F842A4-D5F9-41A7-B328-6D5A02BBE4C9} = {316CDCC7-323F-4264-9FC9-667662BB1F80} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {7BFB3429-B5BB-4DB1-95B4-67D77A864952} diff --git a/playground/DashboardToy/DashboardToy.Common/Class1.cs b/playground/DashboardToy/DashboardToy.Common/Class1.cs new file mode 100644 index 0000000000..adb402e33d --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Common/Class1.cs @@ -0,0 +1,6 @@ +namespace DashboardToy.Common; + +public class Class1 +{ + +} diff --git a/playground/DashboardToy/DashboardToy.Common/DashboardToy.Common.csproj b/playground/DashboardToy/DashboardToy.Common/DashboardToy.Common.csproj new file mode 100644 index 0000000000..09511cc248 --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Common/DashboardToy.Common.csproj @@ -0,0 +1,14 @@ + + + + net8.0 + enable + enable + true + + + + + + + diff --git a/playground/DashboardToy/DashboardToy.Frontend/DashboardToy.Frontend.csproj b/playground/DashboardToy/DashboardToy.Frontend/DashboardToy.Frontend.csproj new file mode 100644 index 0000000000..510c6a3a04 --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/DashboardToy.Frontend.csproj @@ -0,0 +1,19 @@ + + + + net8.0 + enable + enable + true + + + + + + + + + + + + diff --git a/playground/DashboardToy/DashboardToy.Frontend/Data/ClusterDiagnosticsService.cs b/playground/DashboardToy/DashboardToy.Frontend/Data/ClusterDiagnosticsService.cs new file mode 100644 index 0000000000..5126d68992 --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/Data/ClusterDiagnosticsService.cs @@ -0,0 +1,141 @@ +using System.Runtime.InteropServices; +using Orleans.Core.Internal; + +namespace DashboardToy.Frontend.Data; + +public class ClusterDiagnosticsService(IGrainFactory grainFactory) +{ + private readonly Dictionary _hostKeys = []; + private readonly Dictionary _hostDetails = []; + private readonly Dictionary _grainDetails = []; // Grain to host id + private readonly Dictionary _edges = []; + private readonly IManagementGrain _managementGrain = grainFactory.GetGrain(0); + private readonly record struct GrainDetails(int GrainKey, int HostKey); + private readonly record struct HostDetails(int HostKey, int ActivationCount); + private int _version; + + public async ValueTask GetGrainCallFrequencies() + { + var loaderGrain = grainFactory.GetGrain("root"); + var loaderGrainType = loaderGrain.GetGrainId().Type; + var resetCount = await loaderGrain.GetResetCount(); + if (resetCount > _version) + { + _version = resetCount; + await ResetAsync(); + } + + _edges.Clear(); + var maxEdgeValue = 0; + var maxActivationCount = 0; + + var silos = (await _managementGrain.GetHosts(onlyActive: true)).Keys.Order(); + foreach (var silo in silos) + { + var hostKey = GetHostVertex(silo); + var activationCount = 0; + foreach (var activation in await _managementGrain.GetDetailedGrainStatistics(hostsIds: [silo])) + { + if (activation.GrainId.Type.Equals(loaderGrainType)) continue; + if (activation.GrainId.IsSystemTarget()) continue; + var details = GetGrainVertex(activation.GrainId, hostKey); + _grainDetails[activation.GrainId] = new(details.GrainKey, hostKey); + ++activationCount; + } + + maxActivationCount = Math.Max(maxActivationCount, activationCount); + _hostDetails[silo] = new(hostKey, activationCount); + } + + foreach (var edge in await _managementGrain.GetGrainCallFrequencies()) + { + if (edge.TargetGrain.Type.Equals(loaderGrainType) || edge.SourceGrain.Type.Equals(loaderGrainType)) continue; + if (edge.TargetGrain.IsSystemTarget() || edge.SourceGrain.IsSystemTarget()) continue; + var sourceHostId = GetHostVertex(edge.SourceHost); + var targetHostId = GetHostVertex(edge.TargetHost); + var sourceVertex = GetGrainVertex(edge.SourceGrain, sourceHostId); + var targetVertex = GetGrainVertex(edge.TargetGrain, targetHostId); + maxEdgeValue = Math.Max(maxEdgeValue, (int)edge.CallCount); + UpdateEdge(new(sourceVertex.GrainKey, targetVertex.GrainKey), edge.CallCount); + } + + var grainIds = new List(_grainDetails.Count); + CollectionsMarshal.SetCount(grainIds, _grainDetails.Count); + foreach ((var grainId, var (grainKey, hostKey)) in _grainDetails) + { + grainIds[grainKey] = new(grainId.ToString(), grainId.Key.ToString()!, hostKey, 1.0); + } + + var hostIds = new List(_hostKeys.Count); + CollectionsMarshal.SetCount(hostIds, _hostKeys.Count); + foreach ((var hostId, var key) in _hostKeys) + { + var details = _hostDetails[hostId]; + hostIds[key] = new(hostId.ToString(), details.ActivationCount); + } + + var edges = new List(); + + foreach (var edge in _edges) + { + edges.Add(new(edge.Key.Source, edge.Key.Target, edge.Value)); + } + + return new(grainIds, hostIds, edges, maxEdgeValue, maxActivationCount); + } + + internal async ValueTask ResetAsync() + { + var fanoutType = grainFactory.GetGrain(0, "0").GetGrainId().Type; + foreach (var activation in await _managementGrain.GetDetailedGrainStatistics()) + { + if (!activation.GrainId.Type.Equals(fanoutType)) continue; + await grainFactory.GetGrain(activation.GrainId).DeactivateOnIdle(); + } + + Reset(); + } + + internal void Reset() + { + _hostKeys.Clear(); + _hostDetails.Clear(); + _grainDetails.Clear(); + _edges.Clear(); + } + + private GrainDetails GetGrainVertex(GrainId grainId, int hostKey) + { + ref var key = ref CollectionsMarshal.GetValueRefOrAddDefault(_grainDetails, grainId, out var exists); + if (!exists) + { + key = new (_grainDetails.Count - 1, hostKey); + } + + return key; + } + + private int GetHostVertex(SiloAddress silo) + { + ref var key = ref CollectionsMarshal.GetValueRefOrAddDefault(_hostKeys, silo, out var exists); + if (!exists) + { + key = _hostKeys.Count - 1; + } + + return key; + } + + private void UpdateEdge(Key key, ulong increment) + { + ref var count = ref CollectionsMarshal.GetValueRefOrAddDefault(_edges, key, out var exists); + count += increment; + } +} + +public record class CallGraph(List GrainIds, List HostIds, List Edges, int MaxEdgeValue, int MaxActivationCount); + +public record struct HostNode(string Name, int ActivationCount); +public record struct GraphNode(string Name, string Key, int Host, double Weight); +public record struct Key(int Source, int Target); +public record struct GraphEdge(int Source, int Target, double Weight); diff --git a/playground/DashboardToy/DashboardToy.Frontend/Program.cs b/playground/DashboardToy/DashboardToy.Frontend/Program.cs new file mode 100644 index 0000000000..c6f448307e --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/Program.cs @@ -0,0 +1,138 @@ +using DashboardToy.Frontend.Data; +using Microsoft.AspNetCore.Mvc; +using Orleans.Configuration; +using Orleans.Placement.Rebalancing; + +var builder = WebApplication.CreateBuilder(args); +builder.AddKeyedRedisClient("orleans-redis"); +#pragma warning disable ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +builder.UseOrleans(orleans => +{ + orleans.AddActiveRebalancing(); + orleans.Configure(o => + { + o.MinRebalancingPeriod = TimeSpan.FromSeconds(5); + o.MaxRebalancingPeriod = TimeSpan.FromSeconds(15); + o.RecoveryPeriod = TimeSpan.FromSeconds(2); + }); +}); +#pragma warning restore ORLEANSEXP001 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + +// Add services to the container. +builder.Services.AddSingleton(); + +var app = builder.Build(); + +var clusterDiagnosticsService = app.Services.GetRequiredService(); +app.MapGet("/data.json", ([FromServices] ClusterDiagnosticsService clusterDiagnosticsService) => clusterDiagnosticsService.GetGrainCallFrequencies()); +app.MapPost("/reset", async ([FromServices] IGrainFactory grainFactory) => +{ + await grainFactory.GetGrain("root").Reset(); +}); +app.MapPost("/add", async ([FromServices] IGrainFactory grainFactory) => +{ + await grainFactory.GetGrain("root").AddForest(); +}); + +// Configure the HTTP request pipeline. +if (!app.Environment.IsDevelopment()) +{ + app.UseExceptionHandler("/Error"); + // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. + app.UseHsts(); +} + +app.UseHttpsRedirection(); + +app.UseDefaultFiles(); +app.UseStaticFiles(); + +app.UseRouting(); + +await app.StartAsync(); + +var loadGrain = app.Services.GetRequiredService().GetGrain("root"); +await loadGrain.AddForest(); +await loadGrain.AddForest(); +await loadGrain.AddForest(); + +await app.WaitForShutdownAsync(); + +public interface ILoaderGrain : IGrainWithStringKey +{ + ValueTask AddForest(); + ValueTask Reset(); + ValueTask GetResetCount(); +} + +public class LoaderGrain : Grain, ILoaderGrain +{ + private int _numForests = 0; + private int _resetCount; + + public async ValueTask AddForest() + { + var forest = _numForests++; + var loadGrain = GrainFactory.GetGrain(0, forest.ToString()); + await loadGrain.Ping(); + } + + public async ValueTask Reset() + { + ++_resetCount; + _numForests = 0; + await ServiceProvider.GetRequiredService().ResetAsync(); + await GrainFactory.GetGrain(0).ResetGrainCallFrequencies(); + } + + public ValueTask GetResetCount() => new(_resetCount); +} + +public interface IFanOutGrain : IGrainWithIntegerCompoundKey +{ + public ValueTask Ping(); +} + +public class FanOutGrain : Grain, IFanOutGrain +{ + public const int FanOutFactor = 4; + public const int MaxLevel = 2; + private readonly List _children; + + public FanOutGrain() + { + var id = this.GetPrimaryKeyLong(out var forest); + + var level = id == 0 ? 0 : (int)Math.Log(id, FanOutFactor); + var numChildren = level < MaxLevel ? FanOutFactor : 0; + _children = new List(numChildren); + var childBase = (id + 1) * FanOutFactor; + for (var i = 1; i <= numChildren; i++) + { + var child = GrainFactory.GetGrain(childBase + i, forest); + _children.Add(child); + } + + RegisterGrainTimer(() => Ping().AsTask(), TimeSpan.FromSeconds(0.5), TimeSpan.FromSeconds(0.5)); + } + + public async ValueTask Ping() + { + var tasks = new List(_children.Count); + foreach (var child in _children) + { + tasks.Add(child.Ping()); + } + + // Wait for the tasks to complete. + foreach (var task in tasks) + { + await task; + } + } +} + +internal sealed class HardLimitRule : IImbalanceToleranceRule +{ + public bool IsSatisfiedBy(uint imbalance) => imbalance <= 30; +} diff --git a/playground/DashboardToy/DashboardToy.Frontend/Properties/launchSettings.json b/playground/DashboardToy/DashboardToy.Frontend/Properties/launchSettings.json new file mode 100644 index 0000000000..c91c906ed2 --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/Properties/launchSettings.json @@ -0,0 +1,37 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:14770", + "sslPort": 44343 + } + }, + "profiles": { + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:5022", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:7000;http://localhost:5022", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/playground/DashboardToy/DashboardToy.Frontend/appsettings.Development.json b/playground/DashboardToy/DashboardToy.Frontend/appsettings.Development.json new file mode 100644 index 0000000000..770d3e9314 --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "DetailedErrors": true, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/playground/DashboardToy/DashboardToy.Frontend/appsettings.json b/playground/DashboardToy/DashboardToy.Frontend/appsettings.json new file mode 100644 index 0000000000..10f68b8c8b --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "AllowedHosts": "*" +} diff --git a/playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css b/playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css new file mode 100644 index 0000000000..02ae65b5fe --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css @@ -0,0 +1,7 @@ +@charset "UTF-8";/*! + * Bootstrap v5.1.0 (https://getbootstrap.com/) + * Copyright 2011-2021 The Bootstrap Authors + * Copyright 2011-2021 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) + */:root{--bs-blue:#0d6efd;--bs-indigo:#6610f2;--bs-purple:#6f42c1;--bs-pink:#d63384;--bs-red:#dc3545;--bs-orange:#fd7e14;--bs-yellow:#ffc107;--bs-green:#198754;--bs-teal:#20c997;--bs-cyan:#0dcaf0;--bs-white:#fff;--bs-gray:#6c757d;--bs-gray-dark:#343a40;--bs-gray-100:#f8f9fa;--bs-gray-200:#e9ecef;--bs-gray-300:#dee2e6;--bs-gray-400:#ced4da;--bs-gray-500:#adb5bd;--bs-gray-600:#6c757d;--bs-gray-700:#495057;--bs-gray-800:#343a40;--bs-gray-900:#212529;--bs-primary:#0d6efd;--bs-secondary:#6c757d;--bs-success:#198754;--bs-info:#0dcaf0;--bs-warning:#ffc107;--bs-danger:#dc3545;--bs-light:#f8f9fa;--bs-dark:#212529;--bs-primary-rgb:13,110,253;--bs-secondary-rgb:108,117,125;--bs-success-rgb:25,135,84;--bs-info-rgb:13,202,240;--bs-warning-rgb:255,193,7;--bs-danger-rgb:220,53,69;--bs-light-rgb:248,249,250;--bs-dark-rgb:33,37,41;--bs-white-rgb:255,255,255;--bs-black-rgb:0,0,0;--bs-body-rgb:33,37,41;--bs-font-sans-serif:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans","Liberation Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--bs-font-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;--bs-gradient:linear-gradient(180deg, rgba(255, 255, 255, 0.15), rgba(255, 255, 255, 0));--bs-body-font-family:var(--bs-font-sans-serif);--bs-body-font-size:1rem;--bs-body-font-weight:400;--bs-body-line-height:1.5;--bs-body-color:#212529;--bs-body-bg:#fff}*,::after,::before{box-sizing:border-box}@media (prefers-reduced-motion:no-preference){:root{scroll-behavior:smooth}}body{margin:0;font-family:var(--bs-body-font-family);font-size:var(--bs-body-font-size);font-weight:var(--bs-body-font-weight);line-height:var(--bs-body-line-height);color:var(--bs-body-color);text-align:var(--bs-body-text-align);background-color:var(--bs-body-bg);-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}hr{margin:1rem 0;color:inherit;background-color:currentColor;border:0;opacity:.25}hr:not([size]){height:1px}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:calc(1.375rem + 1.5vw)}@media (min-width:1200px){.h1,h1{font-size:2.5rem}}.h2,h2{font-size:calc(1.325rem + .9vw)}@media (min-width:1200px){.h2,h2{font-size:2rem}}.h3,h3{font-size:calc(1.3rem + .6vw)}@media (min-width:1200px){.h3,h3{font-size:1.75rem}}.h4,h4{font-size:calc(1.275rem + .3vw)}@media (min-width:1200px){.h4,h4{font-size:1.5rem}}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}p{margin-top:0;margin-bottom:1rem}abbr[data-bs-original-title],abbr[title]{-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul{padding-left:2rem}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}.small,small{font-size:.875em}.mark,mark{padding:.2em;background-color:#fcf8e3}sub,sup{position:relative;font-size:.75em;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#0d6efd;text-decoration:underline}a:hover{color:#0a58ca}a:not([href]):not([class]),a:not([href]):not([class]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:var(--bs-font-monospace);font-size:1em;direction:ltr;unicode-bidi:bidi-override}pre{display:block;margin-top:0;margin-bottom:1rem;overflow:auto;font-size:.875em}pre code{font-size:inherit;color:inherit;word-break:normal}code{font-size:.875em;color:#d63384;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:.875em;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:1em;font-weight:700}figure{margin:0 0 1rem}img,svg{vertical-align:middle}table{caption-side:bottom;border-collapse:collapse}caption{padding-top:.5rem;padding-bottom:.5rem;color:#6c757d;text-align:left}th{text-align:inherit;text-align:-webkit-match-parent}tbody,td,tfoot,th,thead,tr{border-color:inherit;border-style:solid;border-width:0}label{display:inline-block}button{border-radius:0}button:focus:not(:focus-visible){outline:0}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}select:disabled{opacity:1}[list]::-webkit-calendar-picker-indicator{display:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}::-moz-focus-inner{padding:0;border-style:none}textarea{resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{float:left;width:100%;padding:0;margin-bottom:.5rem;font-size:calc(1.275rem + .3vw);line-height:inherit}@media (min-width:1200px){legend{font-size:1.5rem}}legend+*{clear:left}::-webkit-datetime-edit-day-field,::-webkit-datetime-edit-fields-wrapper,::-webkit-datetime-edit-hour-field,::-webkit-datetime-edit-minute,::-webkit-datetime-edit-month-field,::-webkit-datetime-edit-text,::-webkit-datetime-edit-year-field{padding:0}::-webkit-inner-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:textfield}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-color-swatch-wrapper{padding:0}::file-selector-button{font:inherit}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}iframe{border:0}summary{display:list-item;cursor:pointer}progress{vertical-align:baseline}[hidden]{display:none!important}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:calc(1.625rem + 4.5vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-1{font-size:5rem}}.display-2{font-size:calc(1.575rem + 3.9vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-2{font-size:4.5rem}}.display-3{font-size:calc(1.525rem + 3.3vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-3{font-size:4rem}}.display-4{font-size:calc(1.475rem + 2.7vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-4{font-size:3.5rem}}.display-5{font-size:calc(1.425rem + 2.1vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-5{font-size:3rem}}.display-6{font-size:calc(1.375rem + 1.5vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-6{font-size:2.5rem}}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:.875em;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote>:last-child{margin-bottom:0}.blockquote-footer{margin-top:-1rem;margin-bottom:1rem;font-size:.875em;color:#6c757d}.blockquote-footer::before{content:"— "}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:.875em;color:#6c757d}.container,.container-fluid,.container-lg,.container-md,.container-sm,.container-xl,.container-xxl{width:100%;padding-right:var(--bs-gutter-x,.75rem);padding-left:var(--bs-gutter-x,.75rem);margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}@media (min-width:1400px){.container,.container-lg,.container-md,.container-sm,.container-xl,.container-xxl{max-width:1320px}}.row{--bs-gutter-x:1.5rem;--bs-gutter-y:0;display:flex;flex-wrap:wrap;margin-top:calc(var(--bs-gutter-y) * -1);margin-right:calc(var(--bs-gutter-x) * -.5);margin-left:calc(var(--bs-gutter-x) * -.5)}.row>*{flex-shrink:0;width:100%;max-width:100%;padding-right:calc(var(--bs-gutter-x) * .5);padding-left:calc(var(--bs-gutter-x) * .5);margin-top:var(--bs-gutter-y)}.col{flex:1 0 0%}.row-cols-auto>*{flex:0 0 auto;width:auto}.row-cols-1>*{flex:0 0 auto;width:100%}.row-cols-2>*{flex:0 0 auto;width:50%}.row-cols-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-4>*{flex:0 0 auto;width:25%}.row-cols-5>*{flex:0 0 auto;width:20%}.row-cols-6>*{flex:0 0 auto;width:16.6666666667%}.col-auto{flex:0 0 auto;width:auto}.col-1{flex:0 0 auto;width:8.33333333%}.col-2{flex:0 0 auto;width:16.66666667%}.col-3{flex:0 0 auto;width:25%}.col-4{flex:0 0 auto;width:33.33333333%}.col-5{flex:0 0 auto;width:41.66666667%}.col-6{flex:0 0 auto;width:50%}.col-7{flex:0 0 auto;width:58.33333333%}.col-8{flex:0 0 auto;width:66.66666667%}.col-9{flex:0 0 auto;width:75%}.col-10{flex:0 0 auto;width:83.33333333%}.col-11{flex:0 0 auto;width:91.66666667%}.col-12{flex:0 0 auto;width:100%}.offset-1{margin-left:8.33333333%}.offset-2{margin-left:16.66666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.33333333%}.offset-5{margin-left:41.66666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.33333333%}.offset-8{margin-left:66.66666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.33333333%}.offset-11{margin-left:91.66666667%}.g-0,.gx-0{--bs-gutter-x:0}.g-0,.gy-0{--bs-gutter-y:0}.g-1,.gx-1{--bs-gutter-x:0.25rem}.g-1,.gy-1{--bs-gutter-y:0.25rem}.g-2,.gx-2{--bs-gutter-x:0.5rem}.g-2,.gy-2{--bs-gutter-y:0.5rem}.g-3,.gx-3{--bs-gutter-x:1rem}.g-3,.gy-3{--bs-gutter-y:1rem}.g-4,.gx-4{--bs-gutter-x:1.5rem}.g-4,.gy-4{--bs-gutter-y:1.5rem}.g-5,.gx-5{--bs-gutter-x:3rem}.g-5,.gy-5{--bs-gutter-y:3rem}@media (min-width:576px){.col-sm{flex:1 0 0%}.row-cols-sm-auto>*{flex:0 0 auto;width:auto}.row-cols-sm-1>*{flex:0 0 auto;width:100%}.row-cols-sm-2>*{flex:0 0 auto;width:50%}.row-cols-sm-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-sm-4>*{flex:0 0 auto;width:25%}.row-cols-sm-5>*{flex:0 0 auto;width:20%}.row-cols-sm-6>*{flex:0 0 auto;width:16.6666666667%}.col-sm-auto{flex:0 0 auto;width:auto}.col-sm-1{flex:0 0 auto;width:8.33333333%}.col-sm-2{flex:0 0 auto;width:16.66666667%}.col-sm-3{flex:0 0 auto;width:25%}.col-sm-4{flex:0 0 auto;width:33.33333333%}.col-sm-5{flex:0 0 auto;width:41.66666667%}.col-sm-6{flex:0 0 auto;width:50%}.col-sm-7{flex:0 0 auto;width:58.33333333%}.col-sm-8{flex:0 0 auto;width:66.66666667%}.col-sm-9{flex:0 0 auto;width:75%}.col-sm-10{flex:0 0 auto;width:83.33333333%}.col-sm-11{flex:0 0 auto;width:91.66666667%}.col-sm-12{flex:0 0 auto;width:100%}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.33333333%}.offset-sm-2{margin-left:16.66666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.33333333%}.offset-sm-5{margin-left:41.66666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.33333333%}.offset-sm-8{margin-left:66.66666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.33333333%}.offset-sm-11{margin-left:91.66666667%}.g-sm-0,.gx-sm-0{--bs-gutter-x:0}.g-sm-0,.gy-sm-0{--bs-gutter-y:0}.g-sm-1,.gx-sm-1{--bs-gutter-x:0.25rem}.g-sm-1,.gy-sm-1{--bs-gutter-y:0.25rem}.g-sm-2,.gx-sm-2{--bs-gutter-x:0.5rem}.g-sm-2,.gy-sm-2{--bs-gutter-y:0.5rem}.g-sm-3,.gx-sm-3{--bs-gutter-x:1rem}.g-sm-3,.gy-sm-3{--bs-gutter-y:1rem}.g-sm-4,.gx-sm-4{--bs-gutter-x:1.5rem}.g-sm-4,.gy-sm-4{--bs-gutter-y:1.5rem}.g-sm-5,.gx-sm-5{--bs-gutter-x:3rem}.g-sm-5,.gy-sm-5{--bs-gutter-y:3rem}}@media (min-width:768px){.col-md{flex:1 0 0%}.row-cols-md-auto>*{flex:0 0 auto;width:auto}.row-cols-md-1>*{flex:0 0 auto;width:100%}.row-cols-md-2>*{flex:0 0 auto;width:50%}.row-cols-md-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-md-4>*{flex:0 0 auto;width:25%}.row-cols-md-5>*{flex:0 0 auto;width:20%}.row-cols-md-6>*{flex:0 0 auto;width:16.6666666667%}.col-md-auto{flex:0 0 auto;width:auto}.col-md-1{flex:0 0 auto;width:8.33333333%}.col-md-2{flex:0 0 auto;width:16.66666667%}.col-md-3{flex:0 0 auto;width:25%}.col-md-4{flex:0 0 auto;width:33.33333333%}.col-md-5{flex:0 0 auto;width:41.66666667%}.col-md-6{flex:0 0 auto;width:50%}.col-md-7{flex:0 0 auto;width:58.33333333%}.col-md-8{flex:0 0 auto;width:66.66666667%}.col-md-9{flex:0 0 auto;width:75%}.col-md-10{flex:0 0 auto;width:83.33333333%}.col-md-11{flex:0 0 auto;width:91.66666667%}.col-md-12{flex:0 0 auto;width:100%}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.33333333%}.offset-md-2{margin-left:16.66666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.33333333%}.offset-md-5{margin-left:41.66666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.33333333%}.offset-md-8{margin-left:66.66666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.33333333%}.offset-md-11{margin-left:91.66666667%}.g-md-0,.gx-md-0{--bs-gutter-x:0}.g-md-0,.gy-md-0{--bs-gutter-y:0}.g-md-1,.gx-md-1{--bs-gutter-x:0.25rem}.g-md-1,.gy-md-1{--bs-gutter-y:0.25rem}.g-md-2,.gx-md-2{--bs-gutter-x:0.5rem}.g-md-2,.gy-md-2{--bs-gutter-y:0.5rem}.g-md-3,.gx-md-3{--bs-gutter-x:1rem}.g-md-3,.gy-md-3{--bs-gutter-y:1rem}.g-md-4,.gx-md-4{--bs-gutter-x:1.5rem}.g-md-4,.gy-md-4{--bs-gutter-y:1.5rem}.g-md-5,.gx-md-5{--bs-gutter-x:3rem}.g-md-5,.gy-md-5{--bs-gutter-y:3rem}}@media (min-width:992px){.col-lg{flex:1 0 0%}.row-cols-lg-auto>*{flex:0 0 auto;width:auto}.row-cols-lg-1>*{flex:0 0 auto;width:100%}.row-cols-lg-2>*{flex:0 0 auto;width:50%}.row-cols-lg-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-lg-4>*{flex:0 0 auto;width:25%}.row-cols-lg-5>*{flex:0 0 auto;width:20%}.row-cols-lg-6>*{flex:0 0 auto;width:16.6666666667%}.col-lg-auto{flex:0 0 auto;width:auto}.col-lg-1{flex:0 0 auto;width:8.33333333%}.col-lg-2{flex:0 0 auto;width:16.66666667%}.col-lg-3{flex:0 0 auto;width:25%}.col-lg-4{flex:0 0 auto;width:33.33333333%}.col-lg-5{flex:0 0 auto;width:41.66666667%}.col-lg-6{flex:0 0 auto;width:50%}.col-lg-7{flex:0 0 auto;width:58.33333333%}.col-lg-8{flex:0 0 auto;width:66.66666667%}.col-lg-9{flex:0 0 auto;width:75%}.col-lg-10{flex:0 0 auto;width:83.33333333%}.col-lg-11{flex:0 0 auto;width:91.66666667%}.col-lg-12{flex:0 0 auto;width:100%}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.33333333%}.offset-lg-2{margin-left:16.66666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.33333333%}.offset-lg-5{margin-left:41.66666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.33333333%}.offset-lg-8{margin-left:66.66666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.33333333%}.offset-lg-11{margin-left:91.66666667%}.g-lg-0,.gx-lg-0{--bs-gutter-x:0}.g-lg-0,.gy-lg-0{--bs-gutter-y:0}.g-lg-1,.gx-lg-1{--bs-gutter-x:0.25rem}.g-lg-1,.gy-lg-1{--bs-gutter-y:0.25rem}.g-lg-2,.gx-lg-2{--bs-gutter-x:0.5rem}.g-lg-2,.gy-lg-2{--bs-gutter-y:0.5rem}.g-lg-3,.gx-lg-3{--bs-gutter-x:1rem}.g-lg-3,.gy-lg-3{--bs-gutter-y:1rem}.g-lg-4,.gx-lg-4{--bs-gutter-x:1.5rem}.g-lg-4,.gy-lg-4{--bs-gutter-y:1.5rem}.g-lg-5,.gx-lg-5{--bs-gutter-x:3rem}.g-lg-5,.gy-lg-5{--bs-gutter-y:3rem}}@media (min-width:1200px){.col-xl{flex:1 0 0%}.row-cols-xl-auto>*{flex:0 0 auto;width:auto}.row-cols-xl-1>*{flex:0 0 auto;width:100%}.row-cols-xl-2>*{flex:0 0 auto;width:50%}.row-cols-xl-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-xl-4>*{flex:0 0 auto;width:25%}.row-cols-xl-5>*{flex:0 0 auto;width:20%}.row-cols-xl-6>*{flex:0 0 auto;width:16.6666666667%}.col-xl-auto{flex:0 0 auto;width:auto}.col-xl-1{flex:0 0 auto;width:8.33333333%}.col-xl-2{flex:0 0 auto;width:16.66666667%}.col-xl-3{flex:0 0 auto;width:25%}.col-xl-4{flex:0 0 auto;width:33.33333333%}.col-xl-5{flex:0 0 auto;width:41.66666667%}.col-xl-6{flex:0 0 auto;width:50%}.col-xl-7{flex:0 0 auto;width:58.33333333%}.col-xl-8{flex:0 0 auto;width:66.66666667%}.col-xl-9{flex:0 0 auto;width:75%}.col-xl-10{flex:0 0 auto;width:83.33333333%}.col-xl-11{flex:0 0 auto;width:91.66666667%}.col-xl-12{flex:0 0 auto;width:100%}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.33333333%}.offset-xl-2{margin-left:16.66666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.33333333%}.offset-xl-5{margin-left:41.66666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.33333333%}.offset-xl-8{margin-left:66.66666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.33333333%}.offset-xl-11{margin-left:91.66666667%}.g-xl-0,.gx-xl-0{--bs-gutter-x:0}.g-xl-0,.gy-xl-0{--bs-gutter-y:0}.g-xl-1,.gx-xl-1{--bs-gutter-x:0.25rem}.g-xl-1,.gy-xl-1{--bs-gutter-y:0.25rem}.g-xl-2,.gx-xl-2{--bs-gutter-x:0.5rem}.g-xl-2,.gy-xl-2{--bs-gutter-y:0.5rem}.g-xl-3,.gx-xl-3{--bs-gutter-x:1rem}.g-xl-3,.gy-xl-3{--bs-gutter-y:1rem}.g-xl-4,.gx-xl-4{--bs-gutter-x:1.5rem}.g-xl-4,.gy-xl-4{--bs-gutter-y:1.5rem}.g-xl-5,.gx-xl-5{--bs-gutter-x:3rem}.g-xl-5,.gy-xl-5{--bs-gutter-y:3rem}}@media (min-width:1400px){.col-xxl{flex:1 0 0%}.row-cols-xxl-auto>*{flex:0 0 auto;width:auto}.row-cols-xxl-1>*{flex:0 0 auto;width:100%}.row-cols-xxl-2>*{flex:0 0 auto;width:50%}.row-cols-xxl-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-xxl-4>*{flex:0 0 auto;width:25%}.row-cols-xxl-5>*{flex:0 0 auto;width:20%}.row-cols-xxl-6>*{flex:0 0 auto;width:16.6666666667%}.col-xxl-auto{flex:0 0 auto;width:auto}.col-xxl-1{flex:0 0 auto;width:8.33333333%}.col-xxl-2{flex:0 0 auto;width:16.66666667%}.col-xxl-3{flex:0 0 auto;width:25%}.col-xxl-4{flex:0 0 auto;width:33.33333333%}.col-xxl-5{flex:0 0 auto;width:41.66666667%}.col-xxl-6{flex:0 0 auto;width:50%}.col-xxl-7{flex:0 0 auto;width:58.33333333%}.col-xxl-8{flex:0 0 auto;width:66.66666667%}.col-xxl-9{flex:0 0 auto;width:75%}.col-xxl-10{flex:0 0 auto;width:83.33333333%}.col-xxl-11{flex:0 0 auto;width:91.66666667%}.col-xxl-12{flex:0 0 auto;width:100%}.offset-xxl-0{margin-left:0}.offset-xxl-1{margin-left:8.33333333%}.offset-xxl-2{margin-left:16.66666667%}.offset-xxl-3{margin-left:25%}.offset-xxl-4{margin-left:33.33333333%}.offset-xxl-5{margin-left:41.66666667%}.offset-xxl-6{margin-left:50%}.offset-xxl-7{margin-left:58.33333333%}.offset-xxl-8{margin-left:66.66666667%}.offset-xxl-9{margin-left:75%}.offset-xxl-10{margin-left:83.33333333%}.offset-xxl-11{margin-left:91.66666667%}.g-xxl-0,.gx-xxl-0{--bs-gutter-x:0}.g-xxl-0,.gy-xxl-0{--bs-gutter-y:0}.g-xxl-1,.gx-xxl-1{--bs-gutter-x:0.25rem}.g-xxl-1,.gy-xxl-1{--bs-gutter-y:0.25rem}.g-xxl-2,.gx-xxl-2{--bs-gutter-x:0.5rem}.g-xxl-2,.gy-xxl-2{--bs-gutter-y:0.5rem}.g-xxl-3,.gx-xxl-3{--bs-gutter-x:1rem}.g-xxl-3,.gy-xxl-3{--bs-gutter-y:1rem}.g-xxl-4,.gx-xxl-4{--bs-gutter-x:1.5rem}.g-xxl-4,.gy-xxl-4{--bs-gutter-y:1.5rem}.g-xxl-5,.gx-xxl-5{--bs-gutter-x:3rem}.g-xxl-5,.gy-xxl-5{--bs-gutter-y:3rem}}.table{--bs-table-bg:transparent;--bs-table-accent-bg:transparent;--bs-table-striped-color:#212529;--bs-table-striped-bg:rgba(0, 0, 0, 0.05);--bs-table-active-color:#212529;--bs-table-active-bg:rgba(0, 0, 0, 0.1);--bs-table-hover-color:#212529;--bs-table-hover-bg:rgba(0, 0, 0, 0.075);width:100%;margin-bottom:1rem;color:#212529;vertical-align:top;border-color:#dee2e6}.table>:not(caption)>*>*{padding:.5rem .5rem;background-color:var(--bs-table-bg);border-bottom-width:1px;box-shadow:inset 0 0 0 9999px var(--bs-table-accent-bg)}.table>tbody{vertical-align:inherit}.table>thead{vertical-align:bottom}.table>:not(:last-child)>:last-child>*{border-bottom-color:currentColor}.caption-top{caption-side:top}.table-sm>:not(caption)>*>*{padding:.25rem .25rem}.table-bordered>:not(caption)>*{border-width:1px 0}.table-bordered>:not(caption)>*>*{border-width:0 1px}.table-borderless>:not(caption)>*>*{border-bottom-width:0}.table-striped>tbody>tr:nth-of-type(odd){--bs-table-accent-bg:var(--bs-table-striped-bg);color:var(--bs-table-striped-color)}.table-active{--bs-table-accent-bg:var(--bs-table-active-bg);color:var(--bs-table-active-color)}.table-hover>tbody>tr:hover{--bs-table-accent-bg:var(--bs-table-hover-bg);color:var(--bs-table-hover-color)}.table-primary{--bs-table-bg:#cfe2ff;--bs-table-striped-bg:#c5d7f2;--bs-table-striped-color:#000;--bs-table-active-bg:#bacbe6;--bs-table-active-color:#000;--bs-table-hover-bg:#bfd1ec;--bs-table-hover-color:#000;color:#000;border-color:#bacbe6}.table-secondary{--bs-table-bg:#e2e3e5;--bs-table-striped-bg:#d7d8da;--bs-table-striped-color:#000;--bs-table-active-bg:#cbccce;--bs-table-active-color:#000;--bs-table-hover-bg:#d1d2d4;--bs-table-hover-color:#000;color:#000;border-color:#cbccce}.table-success{--bs-table-bg:#d1e7dd;--bs-table-striped-bg:#c7dbd2;--bs-table-striped-color:#000;--bs-table-active-bg:#bcd0c7;--bs-table-active-color:#000;--bs-table-hover-bg:#c1d6cc;--bs-table-hover-color:#000;color:#000;border-color:#bcd0c7}.table-info{--bs-table-bg:#cff4fc;--bs-table-striped-bg:#c5e8ef;--bs-table-striped-color:#000;--bs-table-active-bg:#badce3;--bs-table-active-color:#000;--bs-table-hover-bg:#bfe2e9;--bs-table-hover-color:#000;color:#000;border-color:#badce3}.table-warning{--bs-table-bg:#fff3cd;--bs-table-striped-bg:#f2e7c3;--bs-table-striped-color:#000;--bs-table-active-bg:#e6dbb9;--bs-table-active-color:#000;--bs-table-hover-bg:#ece1be;--bs-table-hover-color:#000;color:#000;border-color:#e6dbb9}.table-danger{--bs-table-bg:#f8d7da;--bs-table-striped-bg:#eccccf;--bs-table-striped-color:#000;--bs-table-active-bg:#dfc2c4;--bs-table-active-color:#000;--bs-table-hover-bg:#e5c7ca;--bs-table-hover-color:#000;color:#000;border-color:#dfc2c4}.table-light{--bs-table-bg:#f8f9fa;--bs-table-striped-bg:#ecedee;--bs-table-striped-color:#000;--bs-table-active-bg:#dfe0e1;--bs-table-active-color:#000;--bs-table-hover-bg:#e5e6e7;--bs-table-hover-color:#000;color:#000;border-color:#dfe0e1}.table-dark{--bs-table-bg:#212529;--bs-table-striped-bg:#2c3034;--bs-table-striped-color:#fff;--bs-table-active-bg:#373b3e;--bs-table-active-color:#fff;--bs-table-hover-bg:#323539;--bs-table-hover-color:#fff;color:#fff;border-color:#373b3e}.table-responsive{overflow-x:auto;-webkit-overflow-scrolling:touch}@media (max-width:575.98px){.table-responsive-sm{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:767.98px){.table-responsive-md{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:991.98px){.table-responsive-lg{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:1199.98px){.table-responsive-xl{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:1399.98px){.table-responsive-xxl{overflow-x:auto;-webkit-overflow-scrolling:touch}}.form-label{margin-bottom:.5rem}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem}.form-text{margin-top:.25rem;font-size:.875em;color:#6c757d}.form-control{display:block;width:100%;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#212529;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;-webkit-appearance:none;-moz-appearance:none;appearance:none;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control[type=file]{overflow:hidden}.form-control[type=file]:not(:disabled):not([readonly]){cursor:pointer}.form-control:focus{color:#212529;background-color:#fff;border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-control::-webkit-date-and-time-value{height:1.5em}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}.form-control::file-selector-button{padding:.375rem .75rem;margin:-.375rem -.75rem;-webkit-margin-end:.75rem;margin-inline-end:.75rem;color:#212529;background-color:#e9ecef;pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:1px;border-radius:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control::file-selector-button{transition:none}}.form-control:hover:not(:disabled):not([readonly])::file-selector-button{background-color:#dde0e3}.form-control::-webkit-file-upload-button{padding:.375rem .75rem;margin:-.375rem -.75rem;-webkit-margin-end:.75rem;margin-inline-end:.75rem;color:#212529;background-color:#e9ecef;pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:1px;border-radius:0;-webkit-transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control::-webkit-file-upload-button{-webkit-transition:none;transition:none}}.form-control:hover:not(:disabled):not([readonly])::-webkit-file-upload-button{background-color:#dde0e3}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{min-height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.form-control-sm::file-selector-button{padding:.25rem .5rem;margin:-.25rem -.5rem;-webkit-margin-end:.5rem;margin-inline-end:.5rem}.form-control-sm::-webkit-file-upload-button{padding:.25rem .5rem;margin:-.25rem -.5rem;-webkit-margin-end:.5rem;margin-inline-end:.5rem}.form-control-lg{min-height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;border-radius:.3rem}.form-control-lg::file-selector-button{padding:.5rem 1rem;margin:-.5rem -1rem;-webkit-margin-end:1rem;margin-inline-end:1rem}.form-control-lg::-webkit-file-upload-button{padding:.5rem 1rem;margin:-.5rem -1rem;-webkit-margin-end:1rem;margin-inline-end:1rem}textarea.form-control{min-height:calc(1.5em + .75rem + 2px)}textarea.form-control-sm{min-height:calc(1.5em + .5rem + 2px)}textarea.form-control-lg{min-height:calc(1.5em + 1rem + 2px)}.form-control-color{width:3rem;height:auto;padding:.375rem}.form-control-color:not(:disabled):not([readonly]){cursor:pointer}.form-control-color::-moz-color-swatch{height:1.5em;border-radius:.25rem}.form-control-color::-webkit-color-swatch{height:1.5em;border-radius:.25rem}.form-select{display:block;width:100%;padding:.375rem 2.25rem .375rem .75rem;-moz-padding-start:calc(0.75rem - 3px);font-size:1rem;font-weight:400;line-height:1.5;color:#212529;background-color:#fff;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right .75rem center;background-size:16px 12px;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.form-select{transition:none}}.form-select:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-select[multiple],.form-select[size]:not([size="1"]){padding-right:.75rem;background-image:none}.form-select:disabled{background-color:#e9ecef}.form-select:-moz-focusring{color:transparent;text-shadow:0 0 0 #212529}.form-select-sm{padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem}.form-select-lg{padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem}.form-check{display:block;min-height:1.5rem;padding-left:1.5em;margin-bottom:.125rem}.form-check .form-check-input{float:left;margin-left:-1.5em}.form-check-input{width:1em;height:1em;margin-top:.25em;vertical-align:top;background-color:#fff;background-repeat:no-repeat;background-position:center;background-size:contain;border:1px solid rgba(0,0,0,.25);-webkit-appearance:none;-moz-appearance:none;appearance:none;-webkit-print-color-adjust:exact;color-adjust:exact}.form-check-input[type=checkbox]{border-radius:.25em}.form-check-input[type=radio]{border-radius:50%}.form-check-input:active{filter:brightness(90%)}.form-check-input:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-check-input:checked{background-color:#0d6efd;border-color:#0d6efd}.form-check-input:checked[type=checkbox]{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='M6 10l3 3l6-6'/%3e%3c/svg%3e")}.form-check-input:checked[type=radio]{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='2' fill='%23fff'/%3e%3c/svg%3e")}.form-check-input[type=checkbox]:indeterminate{background-color:#0d6efd;border-color:#0d6efd;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='M6 10h8'/%3e%3c/svg%3e")}.form-check-input:disabled{pointer-events:none;filter:none;opacity:.5}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{opacity:.5}.form-switch{padding-left:2.5em}.form-switch .form-check-input{width:2em;margin-left:-2.5em;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%280, 0, 0, 0.25%29'/%3e%3c/svg%3e");background-position:left center;border-radius:2em;transition:background-position .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-switch .form-check-input{transition:none}}.form-switch .form-check-input:focus{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%2386b7fe'/%3e%3c/svg%3e")}.form-switch .form-check-input:checked{background-position:right center;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.form-check-inline{display:inline-block;margin-right:1rem}.btn-check{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.btn-check:disabled+.btn,.btn-check[disabled]+.btn{pointer-events:none;filter:none;opacity:.65}.form-range{width:100%;height:1.5rem;padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.form-range:focus{outline:0}.form-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(13,110,253,.25)}.form-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(13,110,253,.25)}.form-range::-moz-focus-outer{border:0}.form-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#0d6efd;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.form-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.form-range::-webkit-slider-thumb:active{background-color:#b6d4fe}.form-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.form-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#0d6efd;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.form-range::-moz-range-thumb{-moz-transition:none;transition:none}}.form-range::-moz-range-thumb:active{background-color:#b6d4fe}.form-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.form-range:disabled{pointer-events:none}.form-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.form-range:disabled::-moz-range-thumb{background-color:#adb5bd}.form-floating{position:relative}.form-floating>.form-control,.form-floating>.form-select{height:calc(3.5rem + 2px);line-height:1.25}.form-floating>label{position:absolute;top:0;left:0;height:100%;padding:1rem .75rem;pointer-events:none;border:1px solid transparent;transform-origin:0 0;transition:opacity .1s ease-in-out,transform .1s ease-in-out}@media (prefers-reduced-motion:reduce){.form-floating>label{transition:none}}.form-floating>.form-control{padding:1rem .75rem}.form-floating>.form-control::-moz-placeholder{color:transparent}.form-floating>.form-control::placeholder{color:transparent}.form-floating>.form-control:not(:-moz-placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:focus,.form-floating>.form-control:not(:placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:-webkit-autofill{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-select{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:not(:-moz-placeholder-shown)~label{opacity:.65;transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.form-floating>.form-control:focus~label,.form-floating>.form-control:not(:placeholder-shown)~label,.form-floating>.form-select~label{opacity:.65;transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.form-floating>.form-control:-webkit-autofill~label{opacity:.65;transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.input-group{position:relative;display:flex;flex-wrap:wrap;align-items:stretch;width:100%}.input-group>.form-control,.input-group>.form-select{position:relative;flex:1 1 auto;width:1%;min-width:0}.input-group>.form-control:focus,.input-group>.form-select:focus{z-index:3}.input-group .btn{position:relative;z-index:2}.input-group .btn:focus{z-index:3}.input-group-text{display:flex;align-items:center;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-lg>.btn,.input-group-lg>.form-control,.input-group-lg>.form-select,.input-group-lg>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;border-radius:.3rem}.input-group-sm>.btn,.input-group-sm>.form-control,.input-group-sm>.form-select,.input-group-sm>.input-group-text{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.input-group-lg>.form-select,.input-group-sm>.form-select{padding-right:3rem}.input-group:not(.has-validation)>.dropdown-toggle:nth-last-child(n+3),.input-group:not(.has-validation)>:not(:last-child):not(.dropdown-toggle):not(.dropdown-menu){border-top-right-radius:0;border-bottom-right-radius:0}.input-group.has-validation>.dropdown-toggle:nth-last-child(n+4),.input-group.has-validation>:nth-last-child(n+3):not(.dropdown-toggle):not(.dropdown-menu){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>:not(:first-child):not(.dropdown-menu):not(.valid-tooltip):not(.valid-feedback):not(.invalid-tooltip):not(.invalid-feedback){margin-left:-1px;border-top-left-radius:0;border-bottom-left-radius:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:#198754}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:rgba(25,135,84,.9);border-radius:.25rem}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:#198754;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#198754;box-shadow:0 0 0 .25rem rgba(25,135,84,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.form-select.is-valid,.was-validated .form-select:valid{border-color:#198754}.form-select.is-valid:not([multiple]):not([size]),.form-select.is-valid:not([multiple])[size="1"],.was-validated .form-select:valid:not([multiple]):not([size]),.was-validated .form-select:valid:not([multiple])[size="1"]{padding-right:4.125rem;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e"),url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.form-select.is-valid:focus,.was-validated .form-select:valid:focus{border-color:#198754;box-shadow:0 0 0 .25rem rgba(25,135,84,.25)}.form-check-input.is-valid,.was-validated .form-check-input:valid{border-color:#198754}.form-check-input.is-valid:checked,.was-validated .form-check-input:valid:checked{background-color:#198754}.form-check-input.is-valid:focus,.was-validated .form-check-input:valid:focus{box-shadow:0 0 0 .25rem rgba(25,135,84,.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#198754}.form-check-inline .form-check-input~.valid-feedback{margin-left:.5em}.input-group .form-control.is-valid,.input-group .form-select.is-valid,.was-validated .input-group .form-control:valid,.was-validated .input-group .form-select:valid{z-index:1}.input-group .form-control.is-valid:focus,.input-group .form-select.is-valid:focus,.was-validated .input-group .form-control:valid:focus,.was-validated .input-group .form-select:valid:focus{z-index:3}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .25rem rgba(220,53,69,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.form-select.is-invalid,.was-validated .form-select:invalid{border-color:#dc3545}.form-select.is-invalid:not([multiple]):not([size]),.form-select.is-invalid:not([multiple])[size="1"],.was-validated .form-select:invalid:not([multiple]):not([size]),.was-validated .form-select:invalid:not([multiple])[size="1"]{padding-right:4.125rem;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e"),url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.form-select.is-invalid:focus,.was-validated .form-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .25rem rgba(220,53,69,.25)}.form-check-input.is-invalid,.was-validated .form-check-input:invalid{border-color:#dc3545}.form-check-input.is-invalid:checked,.was-validated .form-check-input:invalid:checked{background-color:#dc3545}.form-check-input.is-invalid:focus,.was-validated .form-check-input:invalid:focus{box-shadow:0 0 0 .25rem rgba(220,53,69,.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-inline .form-check-input~.invalid-feedback{margin-left:.5em}.input-group .form-control.is-invalid,.input-group .form-select.is-invalid,.was-validated .input-group .form-control:invalid,.was-validated .input-group .form-select:invalid{z-index:2}.input-group .form-control.is-invalid:focus,.input-group .form-select.is-invalid:focus,.was-validated .input-group .form-control:invalid:focus,.was-validated .input-group .form-select:invalid:focus{z-index:3}.btn{display:inline-block;font-weight:400;line-height:1.5;color:#212529;text-align:center;text-decoration:none;vertical-align:middle;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529}.btn-check:focus+.btn,.btn:focus{outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.btn.disabled,.btn:disabled,fieldset:disabled .btn{pointer-events:none;opacity:.65}.btn-primary{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-primary:hover{color:#fff;background-color:#0b5ed7;border-color:#0a58ca}.btn-check:focus+.btn-primary,.btn-primary:focus{color:#fff;background-color:#0b5ed7;border-color:#0a58ca;box-shadow:0 0 0 .25rem rgba(49,132,253,.5)}.btn-check:active+.btn-primary,.btn-check:checked+.btn-primary,.btn-primary.active,.btn-primary:active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0a58ca;border-color:#0a53be}.btn-check:active+.btn-primary:focus,.btn-check:checked+.btn-primary:focus,.btn-primary.active:focus,.btn-primary:active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(49,132,253,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5c636a;border-color:#565e64}.btn-check:focus+.btn-secondary,.btn-secondary:focus{color:#fff;background-color:#5c636a;border-color:#565e64;box-shadow:0 0 0 .25rem rgba(130,138,145,.5)}.btn-check:active+.btn-secondary,.btn-check:checked+.btn-secondary,.btn-secondary.active,.btn-secondary:active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#565e64;border-color:#51585e}.btn-check:active+.btn-secondary:focus,.btn-check:checked+.btn-secondary:focus,.btn-secondary.active:focus,.btn-secondary:active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-success{color:#fff;background-color:#198754;border-color:#198754}.btn-success:hover{color:#fff;background-color:#157347;border-color:#146c43}.btn-check:focus+.btn-success,.btn-success:focus{color:#fff;background-color:#157347;border-color:#146c43;box-shadow:0 0 0 .25rem rgba(60,153,110,.5)}.btn-check:active+.btn-success,.btn-check:checked+.btn-success,.btn-success.active,.btn-success:active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#146c43;border-color:#13653f}.btn-check:active+.btn-success:focus,.btn-check:checked+.btn-success:focus,.btn-success.active:focus,.btn-success:active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(60,153,110,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#198754;border-color:#198754}.btn-info{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-info:hover{color:#000;background-color:#31d2f2;border-color:#25cff2}.btn-check:focus+.btn-info,.btn-info:focus{color:#000;background-color:#31d2f2;border-color:#25cff2;box-shadow:0 0 0 .25rem rgba(11,172,204,.5)}.btn-check:active+.btn-info,.btn-check:checked+.btn-info,.btn-info.active,.btn-info:active,.show>.btn-info.dropdown-toggle{color:#000;background-color:#3dd5f3;border-color:#25cff2}.btn-check:active+.btn-info:focus,.btn-check:checked+.btn-info:focus,.btn-info.active:focus,.btn-info:active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(11,172,204,.5)}.btn-info.disabled,.btn-info:disabled{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-warning{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#000;background-color:#ffca2c;border-color:#ffc720}.btn-check:focus+.btn-warning,.btn-warning:focus{color:#000;background-color:#ffca2c;border-color:#ffc720;box-shadow:0 0 0 .25rem rgba(217,164,6,.5)}.btn-check:active+.btn-warning,.btn-check:checked+.btn-warning,.btn-warning.active,.btn-warning:active,.show>.btn-warning.dropdown-toggle{color:#000;background-color:#ffcd39;border-color:#ffc720}.btn-check:active+.btn-warning:focus,.btn-check:checked+.btn-warning:focus,.btn-warning.active:focus,.btn-warning:active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(217,164,6,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#bb2d3b;border-color:#b02a37}.btn-check:focus+.btn-danger,.btn-danger:focus{color:#fff;background-color:#bb2d3b;border-color:#b02a37;box-shadow:0 0 0 .25rem rgba(225,83,97,.5)}.btn-check:active+.btn-danger,.btn-check:checked+.btn-danger,.btn-danger.active,.btn-danger:active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#b02a37;border-color:#a52834}.btn-check:active+.btn-danger:focus,.btn-check:checked+.btn-danger:focus,.btn-danger.active:focus,.btn-danger:active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-light{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#000;background-color:#f9fafb;border-color:#f9fafb}.btn-check:focus+.btn-light,.btn-light:focus{color:#000;background-color:#f9fafb;border-color:#f9fafb;box-shadow:0 0 0 .25rem rgba(211,212,213,.5)}.btn-check:active+.btn-light,.btn-check:checked+.btn-light,.btn-light.active,.btn-light:active,.show>.btn-light.dropdown-toggle{color:#000;background-color:#f9fafb;border-color:#f9fafb}.btn-check:active+.btn-light:focus,.btn-check:checked+.btn-light:focus,.btn-light.active:focus,.btn-light:active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(211,212,213,.5)}.btn-light.disabled,.btn-light:disabled{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-dark{color:#fff;background-color:#212529;border-color:#212529}.btn-dark:hover{color:#fff;background-color:#1c1f23;border-color:#1a1e21}.btn-check:focus+.btn-dark,.btn-dark:focus{color:#fff;background-color:#1c1f23;border-color:#1a1e21;box-shadow:0 0 0 .25rem rgba(66,70,73,.5)}.btn-check:active+.btn-dark,.btn-check:checked+.btn-dark,.btn-dark.active,.btn-dark:active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1a1e21;border-color:#191c1f}.btn-check:active+.btn-dark:focus,.btn-check:checked+.btn-dark:focus,.btn-dark.active:focus,.btn-dark:active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(66,70,73,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#212529;border-color:#212529}.btn-outline-primary{color:#0d6efd;border-color:#0d6efd}.btn-outline-primary:hover{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-check:focus+.btn-outline-primary,.btn-outline-primary:focus{box-shadow:0 0 0 .25rem rgba(13,110,253,.5)}.btn-check:active+.btn-outline-primary,.btn-check:checked+.btn-outline-primary,.btn-outline-primary.active,.btn-outline-primary.dropdown-toggle.show,.btn-outline-primary:active{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-check:active+.btn-outline-primary:focus,.btn-check:checked+.btn-outline-primary:focus,.btn-outline-primary.active:focus,.btn-outline-primary.dropdown-toggle.show:focus,.btn-outline-primary:active:focus{box-shadow:0 0 0 .25rem rgba(13,110,253,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#0d6efd;background-color:transparent}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-check:focus+.btn-outline-secondary,.btn-outline-secondary:focus{box-shadow:0 0 0 .25rem rgba(108,117,125,.5)}.btn-check:active+.btn-outline-secondary,.btn-check:checked+.btn-outline-secondary,.btn-outline-secondary.active,.btn-outline-secondary.dropdown-toggle.show,.btn-outline-secondary:active{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-check:active+.btn-outline-secondary:focus,.btn-check:checked+.btn-outline-secondary:focus,.btn-outline-secondary.active:focus,.btn-outline-secondary.dropdown-toggle.show:focus,.btn-outline-secondary:active:focus{box-shadow:0 0 0 .25rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-success{color:#198754;border-color:#198754}.btn-outline-success:hover{color:#fff;background-color:#198754;border-color:#198754}.btn-check:focus+.btn-outline-success,.btn-outline-success:focus{box-shadow:0 0 0 .25rem rgba(25,135,84,.5)}.btn-check:active+.btn-outline-success,.btn-check:checked+.btn-outline-success,.btn-outline-success.active,.btn-outline-success.dropdown-toggle.show,.btn-outline-success:active{color:#fff;background-color:#198754;border-color:#198754}.btn-check:active+.btn-outline-success:focus,.btn-check:checked+.btn-outline-success:focus,.btn-outline-success.active:focus,.btn-outline-success.dropdown-toggle.show:focus,.btn-outline-success:active:focus{box-shadow:0 0 0 .25rem rgba(25,135,84,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#198754;background-color:transparent}.btn-outline-info{color:#0dcaf0;border-color:#0dcaf0}.btn-outline-info:hover{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-check:focus+.btn-outline-info,.btn-outline-info:focus{box-shadow:0 0 0 .25rem rgba(13,202,240,.5)}.btn-check:active+.btn-outline-info,.btn-check:checked+.btn-outline-info,.btn-outline-info.active,.btn-outline-info.dropdown-toggle.show,.btn-outline-info:active{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-check:active+.btn-outline-info:focus,.btn-check:checked+.btn-outline-info:focus,.btn-outline-info.active:focus,.btn-outline-info.dropdown-toggle.show:focus,.btn-outline-info:active:focus{box-shadow:0 0 0 .25rem rgba(13,202,240,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#0dcaf0;background-color:transparent}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-check:focus+.btn-outline-warning,.btn-outline-warning:focus{box-shadow:0 0 0 .25rem rgba(255,193,7,.5)}.btn-check:active+.btn-outline-warning,.btn-check:checked+.btn-outline-warning,.btn-outline-warning.active,.btn-outline-warning.dropdown-toggle.show,.btn-outline-warning:active{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-check:active+.btn-outline-warning:focus,.btn-check:checked+.btn-outline-warning:focus,.btn-outline-warning.active:focus,.btn-outline-warning.dropdown-toggle.show:focus,.btn-outline-warning:active:focus{box-shadow:0 0 0 .25rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-check:focus+.btn-outline-danger,.btn-outline-danger:focus{box-shadow:0 0 0 .25rem rgba(220,53,69,.5)}.btn-check:active+.btn-outline-danger,.btn-check:checked+.btn-outline-danger,.btn-outline-danger.active,.btn-outline-danger.dropdown-toggle.show,.btn-outline-danger:active{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-check:active+.btn-outline-danger:focus,.btn-check:checked+.btn-outline-danger:focus,.btn-outline-danger.active:focus,.btn-outline-danger.dropdown-toggle.show:focus,.btn-outline-danger:active:focus{box-shadow:0 0 0 .25rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-check:focus+.btn-outline-light,.btn-outline-light:focus{box-shadow:0 0 0 .25rem rgba(248,249,250,.5)}.btn-check:active+.btn-outline-light,.btn-check:checked+.btn-outline-light,.btn-outline-light.active,.btn-outline-light.dropdown-toggle.show,.btn-outline-light:active{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-check:active+.btn-outline-light:focus,.btn-check:checked+.btn-outline-light:focus,.btn-outline-light.active:focus,.btn-outline-light.dropdown-toggle.show:focus,.btn-outline-light:active:focus{box-shadow:0 0 0 .25rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-dark{color:#212529;border-color:#212529}.btn-outline-dark:hover{color:#fff;background-color:#212529;border-color:#212529}.btn-check:focus+.btn-outline-dark,.btn-outline-dark:focus{box-shadow:0 0 0 .25rem rgba(33,37,41,.5)}.btn-check:active+.btn-outline-dark,.btn-check:checked+.btn-outline-dark,.btn-outline-dark.active,.btn-outline-dark.dropdown-toggle.show,.btn-outline-dark:active{color:#fff;background-color:#212529;border-color:#212529}.btn-check:active+.btn-outline-dark:focus,.btn-check:checked+.btn-outline-dark:focus,.btn-outline-dark.active:focus,.btn-outline-dark.dropdown-toggle.show:focus,.btn-outline-dark:active:focus{box-shadow:0 0 0 .25rem rgba(33,37,41,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#212529;background-color:transparent}.btn-link{font-weight:400;color:#0d6efd;text-decoration:underline}.btn-link:hover{color:#0a58ca}.btn-link.disabled,.btn-link:disabled{color:#6c757d}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.collapsing.collapse-horizontal{width:0;height:auto;transition:width .35s ease}@media (prefers-reduced-motion:reduce){.collapsing.collapse-horizontal{transition:none}}.dropdown,.dropend,.dropstart,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;z-index:1000;display:none;min-width:10rem;padding:.5rem 0;margin:0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu[data-bs-popper]{top:100%;left:0;margin-top:.125rem}.dropdown-menu-start{--bs-position:start}.dropdown-menu-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-end{--bs-position:end}.dropdown-menu-end[data-bs-popper]{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-start{--bs-position:start}.dropdown-menu-sm-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-sm-end{--bs-position:end}.dropdown-menu-sm-end[data-bs-popper]{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-start{--bs-position:start}.dropdown-menu-md-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-md-end{--bs-position:end}.dropdown-menu-md-end[data-bs-popper]{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-start{--bs-position:start}.dropdown-menu-lg-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-lg-end{--bs-position:end}.dropdown-menu-lg-end[data-bs-popper]{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-start{--bs-position:start}.dropdown-menu-xl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xl-end{--bs-position:end}.dropdown-menu-xl-end[data-bs-popper]{right:0;left:auto}}@media (min-width:1400px){.dropdown-menu-xxl-start{--bs-position:start}.dropdown-menu-xxl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xxl-end{--bs-position:end}.dropdown-menu-xxl-end[data-bs-popper]{right:0;left:auto}}.dropup .dropdown-menu[data-bs-popper]{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-menu[data-bs-popper]{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropend .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropend .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-toggle::after{vertical-align:0}.dropstart .dropdown-menu[data-bs-popper]{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropstart .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropstart .dropdown-toggle::after{display:none}.dropstart .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropstart .dropdown-toggle:empty::after{margin-left:0}.dropstart .dropdown-toggle::before{vertical-align:0}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid rgba(0,0,0,.15)}.dropdown-item{display:block;width:100%;padding:.25rem 1rem;clear:both;font-weight:400;color:#212529;text-align:inherit;text-decoration:none;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#1e2125;background-color:#e9ecef}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#0d6efd}.dropdown-item.disabled,.dropdown-item:disabled{color:#adb5bd;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1rem;color:#212529}.dropdown-menu-dark{color:#dee2e6;background-color:#343a40;border-color:rgba(0,0,0,.15)}.dropdown-menu-dark .dropdown-item{color:#dee2e6}.dropdown-menu-dark .dropdown-item:focus,.dropdown-menu-dark .dropdown-item:hover{color:#fff;background-color:rgba(255,255,255,.15)}.dropdown-menu-dark .dropdown-item.active,.dropdown-menu-dark .dropdown-item:active{color:#fff;background-color:#0d6efd}.dropdown-menu-dark .dropdown-item.disabled,.dropdown-menu-dark .dropdown-item:disabled{color:#adb5bd}.dropdown-menu-dark .dropdown-divider{border-color:rgba(0,0,0,.15)}.dropdown-menu-dark .dropdown-item-text{color:#dee2e6}.dropdown-menu-dark .dropdown-header{color:#adb5bd}.btn-group,.btn-group-vertical{position:relative;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;flex:1 1 auto}.btn-group-vertical>.btn-check:checked+.btn,.btn-group-vertical>.btn-check:focus+.btn,.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:hover,.btn-group>.btn-check:checked+.btn,.btn-group>.btn-check:focus+.btn,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus,.btn-group>.btn:hover{z-index:1}.btn-toolbar{display:flex;flex-wrap:wrap;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:nth-child(n+3),.btn-group>:not(.btn-check)+.btn{border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropend .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropstart .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{flex-direction:column;align-items:flex-start;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn~.btn{border-top-left-radius:0;border-top-right-radius:0}.nav{display:flex;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem;color:#0d6efd;text-decoration:none;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out}@media (prefers-reduced-motion:reduce){.nav-link{transition:none}}.nav-link:focus,.nav-link:hover{color:#0a58ca}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-link{margin-bottom:-1px;background:0 0;border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6;isolation:isolate}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{background:0 0;border:0;border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#0d6efd}.nav-fill .nav-item,.nav-fill>.nav-link{flex:1 1 auto;text-align:center}.nav-justified .nav-item,.nav-justified>.nav-link{flex-basis:0;flex-grow:1;text-align:center}.nav-fill .nav-item .nav-link,.nav-justified .nav-item .nav-link{width:100%}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;padding-top:.5rem;padding-bottom:.5rem}.navbar>.container,.navbar>.container-fluid,.navbar>.container-lg,.navbar>.container-md,.navbar>.container-sm,.navbar>.container-xl,.navbar>.container-xxl{display:flex;flex-wrap:inherit;align-items:center;justify-content:space-between}.navbar-brand{padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;text-decoration:none;white-space:nowrap}.navbar-nav{display:flex;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static}.navbar-text{padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{flex-basis:100%;flex-grow:1;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem;transition:box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.navbar-toggler{transition:none}}.navbar-toggler:hover{text-decoration:none}.navbar-toggler:focus{text-decoration:none;outline:0;box-shadow:0 0 0 .25rem}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;background-repeat:no-repeat;background-position:center;background-size:100%}.navbar-nav-scroll{max-height:var(--bs-scroll-height,75vh);overflow-y:auto}@media (min-width:576px){.navbar-expand-sm{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-sm .navbar-nav{flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm .navbar-nav-scroll{overflow:visible}.navbar-expand-sm .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}.navbar-expand-sm .offcanvas-header{display:none}.navbar-expand-sm .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-sm .offcanvas-bottom,.navbar-expand-sm .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-sm .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:768px){.navbar-expand-md{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-md .navbar-nav{flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md .navbar-nav-scroll{overflow:visible}.navbar-expand-md .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}.navbar-expand-md .offcanvas-header{display:none}.navbar-expand-md .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-md .offcanvas-bottom,.navbar-expand-md .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-md .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:992px){.navbar-expand-lg{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-lg .navbar-nav{flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg .navbar-nav-scroll{overflow:visible}.navbar-expand-lg .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}.navbar-expand-lg .offcanvas-header{display:none}.navbar-expand-lg .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-lg .offcanvas-bottom,.navbar-expand-lg .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-lg .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:1200px){.navbar-expand-xl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xl .navbar-nav{flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl .navbar-nav-scroll{overflow:visible}.navbar-expand-xl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}.navbar-expand-xl .offcanvas-header{display:none}.navbar-expand-xl .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-xl .offcanvas-bottom,.navbar-expand-xl .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-xl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:1400px){.navbar-expand-xxl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xxl .navbar-nav{flex-direction:row}.navbar-expand-xxl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xxl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xxl .navbar-nav-scroll{overflow:visible}.navbar-expand-xxl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xxl .navbar-toggler{display:none}.navbar-expand-xxl .offcanvas-header{display:none}.navbar-expand-xxl .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-xxl .offcanvas-bottom,.navbar-expand-xxl .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-xxl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}.navbar-expand{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand .navbar-nav{flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand .navbar-nav-scroll{overflow:visible}.navbar-expand .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-expand .offcanvas-header{display:none}.navbar-expand .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand .offcanvas-bottom,.navbar-expand .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.55)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.55);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%280, 0, 0, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.55)}.navbar-light .navbar-text a,.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.55)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.55);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.55)}.navbar-dark .navbar-text a,.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:flex;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card>.card-header+.list-group,.card>.list-group+.card-footer{border-top:0}.card-body{flex:1 1 auto;padding:1rem 1rem}.card-title{margin-bottom:.5rem}.card-subtitle{margin-top:-.25rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link+.card-link{margin-left:1rem}.card-header{padding:.5rem 1rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-footer{padding:.5rem 1rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.5rem;margin-bottom:-.5rem;margin-left:-.5rem;border-bottom:0}.card-header-pills{margin-right:-.5rem;margin-left:-.5rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1rem;border-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom,.card-img-top{width:100%}.card-img,.card-img-top{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-group>.card{margin-bottom:.75rem}@media (min-width:576px){.card-group{display:flex;flex-flow:row wrap}.card-group>.card{flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.accordion-button{position:relative;display:flex;align-items:center;width:100%;padding:1rem 1.25rem;font-size:1rem;color:#212529;text-align:left;background-color:#fff;border:0;border-radius:0;overflow-anchor:none;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,border-radius .15s ease}@media (prefers-reduced-motion:reduce){.accordion-button{transition:none}}.accordion-button:not(.collapsed){color:#0c63e4;background-color:#e7f1ff;box-shadow:inset 0 -1px 0 rgba(0,0,0,.125)}.accordion-button:not(.collapsed)::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%230c63e4'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");transform:rotate(-180deg)}.accordion-button::after{flex-shrink:0;width:1.25rem;height:1.25rem;margin-left:auto;content:"";background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23212529'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-size:1.25rem;transition:transform .2s ease-in-out}@media (prefers-reduced-motion:reduce){.accordion-button::after{transition:none}}.accordion-button:hover{z-index:2}.accordion-button:focus{z-index:3;border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.accordion-header{margin-bottom:0}.accordion-item{background-color:#fff;border:1px solid rgba(0,0,0,.125)}.accordion-item:first-of-type{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.accordion-item:first-of-type .accordion-button{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.accordion-item:not(:first-of-type){border-top:0}.accordion-item:last-of-type{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.accordion-item:last-of-type .accordion-button.collapsed{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.accordion-item:last-of-type .accordion-collapse{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.accordion-body{padding:1rem 1.25rem}.accordion-flush .accordion-collapse{border-width:0}.accordion-flush .accordion-item{border-right:0;border-left:0;border-radius:0}.accordion-flush .accordion-item:first-child{border-top:0}.accordion-flush .accordion-item:last-child{border-bottom:0}.accordion-flush .accordion-item .accordion-button{border-radius:0}.breadcrumb{display:flex;flex-wrap:wrap;padding:0 0;margin-bottom:1rem;list-style:none}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{float:left;padding-right:.5rem;color:#6c757d;content:var(--bs-breadcrumb-divider, "/")}.breadcrumb-item.active{color:#6c757d}.pagination{display:flex;padding-left:0;list-style:none}.page-link{position:relative;display:block;color:#0d6efd;text-decoration:none;background-color:#fff;border:1px solid #dee2e6;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.page-link{transition:none}}.page-link:hover{z-index:2;color:#0a58ca;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:3;color:#0a58ca;background-color:#e9ecef;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.page-item:not(:first-child) .page-link{margin-left:-1px}.page-item.active .page-link{z-index:3;color:#fff;background-color:#0d6efd;border-color:#0d6efd}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;background-color:#fff;border-color:#dee2e6}.page-link{padding:.375rem .75rem}.page-item:first-child .page-link{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.35em .65em;font-size:.75em;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.alert{position:relative;padding:1rem 1rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:3rem}.alert-dismissible .btn-close{position:absolute;top:0;right:0;z-index:2;padding:1.25rem 1rem}.alert-primary{color:#084298;background-color:#cfe2ff;border-color:#b6d4fe}.alert-primary .alert-link{color:#06357a}.alert-secondary{color:#41464b;background-color:#e2e3e5;border-color:#d3d6d8}.alert-secondary .alert-link{color:#34383c}.alert-success{color:#0f5132;background-color:#d1e7dd;border-color:#badbcc}.alert-success .alert-link{color:#0c4128}.alert-info{color:#055160;background-color:#cff4fc;border-color:#b6effb}.alert-info .alert-link{color:#04414d}.alert-warning{color:#664d03;background-color:#fff3cd;border-color:#ffecb5}.alert-warning .alert-link{color:#523e02}.alert-danger{color:#842029;background-color:#f8d7da;border-color:#f5c2c7}.alert-danger .alert-link{color:#6a1a21}.alert-light{color:#636464;background-color:#fefefe;border-color:#fdfdfe}.alert-light .alert-link{color:#4f5050}.alert-dark{color:#141619;background-color:#d3d3d4;border-color:#bcbebf}.alert-dark .alert-link{color:#101214}@-webkit-keyframes progress-bar-stripes{0%{background-position-x:1rem}}@keyframes progress-bar-stripes{0%{background-position-x:1rem}}.progress{display:flex;height:1rem;overflow:hidden;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:flex;flex-direction:column;justify-content:center;overflow:hidden;color:#fff;text-align:center;white-space:nowrap;background-color:#0d6efd;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:1s linear infinite progress-bar-stripes;animation:1s linear infinite progress-bar-stripes}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.list-group{display:flex;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:.25rem}.list-group-numbered{list-style-type:none;counter-reset:section}.list-group-numbered>li::before{content:counters(section, ".") ". ";counter-increment:section}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.5rem 1rem;color:#212529;text-decoration:none;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#0d6efd;border-color:#0d6efd}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:-1px;border-top-width:1px}.list-group-horizontal{flex-direction:row}.list-group-horizontal>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}@media (min-width:576px){.list-group-horizontal-sm{flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:768px){.list-group-horizontal-md{flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:992px){.list-group-horizontal-lg{flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1200px){.list-group-horizontal-xl{flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1400px){.list-group-horizontal-xxl{flex-direction:row}.list-group-horizontal-xxl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xxl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xxl>.list-group-item.active{margin-top:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 1px}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{color:#084298;background-color:#cfe2ff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#084298;background-color:#bacbe6}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#084298;border-color:#084298}.list-group-item-secondary{color:#41464b;background-color:#e2e3e5}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#41464b;background-color:#cbccce}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#41464b;border-color:#41464b}.list-group-item-success{color:#0f5132;background-color:#d1e7dd}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#0f5132;background-color:#bcd0c7}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#0f5132;border-color:#0f5132}.list-group-item-info{color:#055160;background-color:#cff4fc}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#055160;background-color:#badce3}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#055160;border-color:#055160}.list-group-item-warning{color:#664d03;background-color:#fff3cd}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#664d03;background-color:#e6dbb9}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#664d03;border-color:#664d03}.list-group-item-danger{color:#842029;background-color:#f8d7da}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#842029;background-color:#dfc2c4}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#842029;border-color:#842029}.list-group-item-light{color:#636464;background-color:#fefefe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#636464;background-color:#e5e5e5}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#636464;border-color:#636464}.list-group-item-dark{color:#141619;background-color:#d3d3d4}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#141619;background-color:#bebebf}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#141619;border-color:#141619}.btn-close{box-sizing:content-box;width:1em;height:1em;padding:.25em .25em;color:#000;background:transparent url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23000'%3e%3cpath d='M.293.293a1 1 0 011.414 0L8 6.586 14.293.293a1 1 0 111.414 1.414L9.414 8l6.293 6.293a1 1 0 01-1.414 1.414L8 9.414l-6.293 6.293a1 1 0 01-1.414-1.414L6.586 8 .293 1.707a1 1 0 010-1.414z'/%3e%3c/svg%3e") center/1em auto no-repeat;border:0;border-radius:.25rem;opacity:.5}.btn-close:hover{color:#000;text-decoration:none;opacity:.75}.btn-close:focus{outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25);opacity:1}.btn-close.disabled,.btn-close:disabled{pointer-events:none;-webkit-user-select:none;-moz-user-select:none;user-select:none;opacity:.25}.btn-close-white{filter:invert(1) grayscale(100%) brightness(200%)}.toast{width:350px;max-width:100%;font-size:.875rem;pointer-events:auto;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .5rem 1rem rgba(0,0,0,.15);border-radius:.25rem}.toast.showing{opacity:0}.toast:not(.show){display:none}.toast-container{width:-webkit-max-content;width:-moz-max-content;width:max-content;max-width:100%;pointer-events:none}.toast-container>:not(:last-child){margin-bottom:.75rem}.toast-header{display:flex;align-items:center;padding:.5rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05);border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.toast-header .btn-close{margin-right:-.375rem;margin-left:.75rem}.toast-body{padding:.75rem;word-wrap:break-word}.modal{position:fixed;top:0;left:0;z-index:1055;display:none;width:100%;height:100%;overflow-x:hidden;overflow-y:auto;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:transform .3s ease-out;transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{transform:none}.modal.modal-static .modal-dialog{transform:scale(1.02)}.modal-dialog-scrollable{height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:100%;overflow:hidden}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:flex;align-items:center;min-height:calc(100% - 1rem)}.modal-content{position:relative;display:flex;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1050;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:flex;flex-shrink:0;align-items:center;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.modal-header .btn-close{padding:.5rem .5rem;margin:-.5rem -.5rem -.5rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;flex:1 1 auto;padding:1rem}.modal-footer{display:flex;flex-wrap:wrap;flex-shrink:0;align-items:center;justify-content:flex-end;padding:.75rem;border-top:1px solid #dee2e6;border-bottom-right-radius:calc(.3rem - 1px);border-bottom-left-radius:calc(.3rem - 1px)}.modal-footer>*{margin:.25rem}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{height:calc(100% - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.modal-fullscreen{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen .modal-header{border-radius:0}.modal-fullscreen .modal-body{overflow-y:auto}.modal-fullscreen .modal-footer{border-radius:0}@media (max-width:575.98px){.modal-fullscreen-sm-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-sm-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-sm-down .modal-header{border-radius:0}.modal-fullscreen-sm-down .modal-body{overflow-y:auto}.modal-fullscreen-sm-down .modal-footer{border-radius:0}}@media (max-width:767.98px){.modal-fullscreen-md-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-md-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-md-down .modal-header{border-radius:0}.modal-fullscreen-md-down .modal-body{overflow-y:auto}.modal-fullscreen-md-down .modal-footer{border-radius:0}}@media (max-width:991.98px){.modal-fullscreen-lg-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-lg-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-lg-down .modal-header{border-radius:0}.modal-fullscreen-lg-down .modal-body{overflow-y:auto}.modal-fullscreen-lg-down .modal-footer{border-radius:0}}@media (max-width:1199.98px){.modal-fullscreen-xl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xl-down .modal-header{border-radius:0}.modal-fullscreen-xl-down .modal-body{overflow-y:auto}.modal-fullscreen-xl-down .modal-footer{border-radius:0}}@media (max-width:1399.98px){.modal-fullscreen-xxl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xxl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xxl-down .modal-header{border-radius:0}.modal-fullscreen-xxl-down .modal-body{overflow-y:auto}.modal-fullscreen-xxl-down .modal-footer{border-radius:0}}.tooltip{position:absolute;z-index:1080;display:block;margin:0;font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .tooltip-arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .tooltip-arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[data-popper-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow,.bs-tooltip-top .tooltip-arrow{bottom:0}.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow::before,.bs-tooltip-top .tooltip-arrow::before{top:-1px;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[data-popper-placement^=right],.bs-tooltip-end{padding:0 .4rem}.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow,.bs-tooltip-end .tooltip-arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow::before,.bs-tooltip-end .tooltip-arrow::before{right:-1px;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[data-popper-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow,.bs-tooltip-bottom .tooltip-arrow{top:0}.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow::before,.bs-tooltip-bottom .tooltip-arrow::before{bottom:-1px;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[data-popper-placement^=left],.bs-tooltip-start{padding:0 .4rem}.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow,.bs-tooltip-start .tooltip-arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow::before,.bs-tooltip-start .tooltip-arrow::before{left:-1px;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1070;display:block;max-width:276px;font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .popover-arrow{position:absolute;display:block;width:1rem;height:.5rem}.popover .popover-arrow::after,.popover .popover-arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow,.bs-popover-top>.popover-arrow{bottom:calc(-.5rem - 1px)}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::before,.bs-popover-top>.popover-arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::after,.bs-popover-top>.popover-arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow,.bs-popover-end>.popover-arrow{left:calc(-.5rem - 1px);width:.5rem;height:1rem}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::before,.bs-popover-end>.popover-arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::after,.bs-popover-end>.popover-arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow,.bs-popover-bottom>.popover-arrow{top:calc(-.5rem - 1px)}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::before,.bs-popover-bottom>.popover-arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::after,.bs-popover-bottom>.popover-arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[data-popper-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f0f0f0}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow,.bs-popover-start>.popover-arrow{right:calc(-.5rem - 1px);width:.5rem;height:1rem}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::before,.bs-popover-start>.popover-arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::after,.bs-popover-start>.popover-arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem 1rem;margin-bottom:0;font-size:1rem;background-color:#f0f0f0;border-bottom:1px solid rgba(0,0,0,.2);border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:1rem 1rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-end,.carousel-item-next:not(.carousel-item-start){transform:translateX(100%)}.active.carousel-item-start,.carousel-item-prev:not(.carousel-item-end){transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;transform:none}.carousel-fade .carousel-item-next.carousel-item-start,.carousel-fade .carousel-item-prev.carousel-item-end,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-end,.carousel-fade .active.carousel-item-start{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-end,.carousel-fade .active.carousel-item-start{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:flex;align-items:center;justify-content:center;width:15%;padding:0;color:#fff;text-align:center;background:0 0;border:0;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:2rem;height:2rem;background-repeat:no-repeat;background-position:50%;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M11.354 1.646a.5.5 0 0 1 0 .708L5.707 8l5.647 5.646a.5.5 0 0 1-.708.708l-6-6a.5.5 0 0 1 0-.708l6-6a.5.5 0 0 1 .708 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M4.646 1.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1 0 .708l-6 6a.5.5 0 0 1-.708-.708L10.293 8 4.646 2.354a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:2;display:flex;justify-content:center;padding:0;margin-right:15%;margin-bottom:1rem;margin-left:15%;list-style:none}.carousel-indicators [data-bs-target]{box-sizing:content-box;flex:0 1 auto;width:30px;height:3px;padding:0;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border:0;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators [data-bs-target]{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:1.25rem;left:15%;padding-top:1.25rem;padding-bottom:1.25rem;color:#fff;text-align:center}.carousel-dark .carousel-control-next-icon,.carousel-dark .carousel-control-prev-icon{filter:invert(1) grayscale(100)}.carousel-dark .carousel-indicators [data-bs-target]{background-color:#000}.carousel-dark .carousel-caption{color:#000}@-webkit-keyframes spinner-border{to{transform:rotate(360deg)}}@keyframes spinner-border{to{transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:-.125em;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:.75s linear infinite spinner-border;animation:.75s linear infinite spinner-border}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1;transform:none}}@keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1;transform:none}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:-.125em;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:.75s linear infinite spinner-grow;animation:.75s linear infinite spinner-grow}.spinner-grow-sm{width:1rem;height:1rem}@media (prefers-reduced-motion:reduce){.spinner-border,.spinner-grow{-webkit-animation-duration:1.5s;animation-duration:1.5s}}.offcanvas{position:fixed;bottom:0;z-index:1045;display:flex;flex-direction:column;max-width:100%;visibility:hidden;background-color:#fff;background-clip:padding-box;outline:0;transition:transform .3s ease-in-out}@media (prefers-reduced-motion:reduce){.offcanvas{transition:none}}.offcanvas-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.offcanvas-backdrop.fade{opacity:0}.offcanvas-backdrop.show{opacity:.5}.offcanvas-header{display:flex;align-items:center;justify-content:space-between;padding:1rem 1rem}.offcanvas-header .btn-close{padding:.5rem .5rem;margin-top:-.5rem;margin-right:-.5rem;margin-bottom:-.5rem}.offcanvas-title{margin-bottom:0;line-height:1.5}.offcanvas-body{flex-grow:1;padding:1rem 1rem;overflow-y:auto}.offcanvas-start{top:0;left:0;width:400px;border-right:1px solid rgba(0,0,0,.2);transform:translateX(-100%)}.offcanvas-end{top:0;right:0;width:400px;border-left:1px solid rgba(0,0,0,.2);transform:translateX(100%)}.offcanvas-top{top:0;right:0;left:0;height:30vh;max-height:100%;border-bottom:1px solid rgba(0,0,0,.2);transform:translateY(-100%)}.offcanvas-bottom{right:0;left:0;height:30vh;max-height:100%;border-top:1px solid rgba(0,0,0,.2);transform:translateY(100%)}.offcanvas.show{transform:none}.placeholder{display:inline-block;min-height:1em;vertical-align:middle;cursor:wait;background-color:currentColor;opacity:.5}.placeholder.btn::before{display:inline-block;content:""}.placeholder-xs{min-height:.6em}.placeholder-sm{min-height:.8em}.placeholder-lg{min-height:1.2em}.placeholder-glow .placeholder{-webkit-animation:placeholder-glow 2s ease-in-out infinite;animation:placeholder-glow 2s ease-in-out infinite}@-webkit-keyframes placeholder-glow{50%{opacity:.2}}@keyframes placeholder-glow{50%{opacity:.2}}.placeholder-wave{-webkit-mask-image:linear-gradient(130deg,#000 55%,rgba(0,0,0,0.8) 75%,#000 95%);mask-image:linear-gradient(130deg,#000 55%,rgba(0,0,0,0.8) 75%,#000 95%);-webkit-mask-size:200% 100%;mask-size:200% 100%;-webkit-animation:placeholder-wave 2s linear infinite;animation:placeholder-wave 2s linear infinite}@-webkit-keyframes placeholder-wave{100%{-webkit-mask-position:-200% 0%;mask-position:-200% 0%}}@keyframes placeholder-wave{100%{-webkit-mask-position:-200% 0%;mask-position:-200% 0%}}.clearfix::after{display:block;clear:both;content:""}.link-primary{color:#0d6efd}.link-primary:focus,.link-primary:hover{color:#0a58ca}.link-secondary{color:#6c757d}.link-secondary:focus,.link-secondary:hover{color:#565e64}.link-success{color:#198754}.link-success:focus,.link-success:hover{color:#146c43}.link-info{color:#0dcaf0}.link-info:focus,.link-info:hover{color:#3dd5f3}.link-warning{color:#ffc107}.link-warning:focus,.link-warning:hover{color:#ffcd39}.link-danger{color:#dc3545}.link-danger:focus,.link-danger:hover{color:#b02a37}.link-light{color:#f8f9fa}.link-light:focus,.link-light:hover{color:#f9fafb}.link-dark{color:#212529}.link-dark:focus,.link-dark:hover{color:#1a1e21}.ratio{position:relative;width:100%}.ratio::before{display:block;padding-top:var(--bs-aspect-ratio);content:""}.ratio>*{position:absolute;top:0;left:0;width:100%;height:100%}.ratio-1x1{--bs-aspect-ratio:100%}.ratio-4x3{--bs-aspect-ratio:calc(3 / 4 * 100%)}.ratio-16x9{--bs-aspect-ratio:calc(9 / 16 * 100%)}.ratio-21x9{--bs-aspect-ratio:calc(9 / 21 * 100%)}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}@media (min-width:576px){.sticky-sm-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:768px){.sticky-md-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:992px){.sticky-lg-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:1200px){.sticky-xl-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:1400px){.sticky-xxl-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.hstack{display:flex;flex-direction:row;align-items:center;align-self:stretch}.vstack{display:flex;flex:1 1 auto;flex-direction:column;align-self:stretch}.visually-hidden,.visually-hidden-focusable:not(:focus):not(:focus-within){position:absolute!important;width:1px!important;height:1px!important;padding:0!important;margin:-1px!important;overflow:hidden!important;clip:rect(0,0,0,0)!important;white-space:nowrap!important;border:0!important}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;content:""}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.vr{display:inline-block;align-self:stretch;width:1px;min-height:1em;background-color:currentColor;opacity:.25}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.float-start{float:left!important}.float-end{float:right!important}.float-none{float:none!important}.opacity-0{opacity:0!important}.opacity-25{opacity:.25!important}.opacity-50{opacity:.5!important}.opacity-75{opacity:.75!important}.opacity-100{opacity:1!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.overflow-visible{overflow:visible!important}.overflow-scroll{overflow:scroll!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-grid{display:grid!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:flex!important}.d-inline-flex{display:inline-flex!important}.d-none{display:none!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.top-0{top:0!important}.top-50{top:50%!important}.top-100{top:100%!important}.bottom-0{bottom:0!important}.bottom-50{bottom:50%!important}.bottom-100{bottom:100%!important}.start-0{left:0!important}.start-50{left:50%!important}.start-100{left:100%!important}.end-0{right:0!important}.end-50{right:50%!important}.end-100{right:100%!important}.translate-middle{transform:translate(-50%,-50%)!important}.translate-middle-x{transform:translateX(-50%)!important}.translate-middle-y{transform:translateY(-50%)!important}.border{border:1px solid #dee2e6!important}.border-0{border:0!important}.border-top{border-top:1px solid #dee2e6!important}.border-top-0{border-top:0!important}.border-end{border-right:1px solid #dee2e6!important}.border-end-0{border-right:0!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-bottom-0{border-bottom:0!important}.border-start{border-left:1px solid #dee2e6!important}.border-start-0{border-left:0!important}.border-primary{border-color:#0d6efd!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#198754!important}.border-info{border-color:#0dcaf0!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#212529!important}.border-white{border-color:#fff!important}.border-1{border-width:1px!important}.border-2{border-width:2px!important}.border-3{border-width:3px!important}.border-4{border-width:4px!important}.border-5{border-width:5px!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.mw-100{max-width:100%!important}.vw-100{width:100vw!important}.min-vw-100{min-width:100vw!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mh-100{max-height:100%!important}.vh-100{height:100vh!important}.min-vh-100{min-height:100vh!important}.flex-fill{flex:1 1 auto!important}.flex-row{flex-direction:row!important}.flex-column{flex-direction:column!important}.flex-row-reverse{flex-direction:row-reverse!important}.flex-column-reverse{flex-direction:column-reverse!important}.flex-grow-0{flex-grow:0!important}.flex-grow-1{flex-grow:1!important}.flex-shrink-0{flex-shrink:0!important}.flex-shrink-1{flex-shrink:1!important}.flex-wrap{flex-wrap:wrap!important}.flex-nowrap{flex-wrap:nowrap!important}.flex-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-0{gap:0!important}.gap-1{gap:.25rem!important}.gap-2{gap:.5rem!important}.gap-3{gap:1rem!important}.gap-4{gap:1.5rem!important}.gap-5{gap:3rem!important}.justify-content-start{justify-content:flex-start!important}.justify-content-end{justify-content:flex-end!important}.justify-content-center{justify-content:center!important}.justify-content-between{justify-content:space-between!important}.justify-content-around{justify-content:space-around!important}.justify-content-evenly{justify-content:space-evenly!important}.align-items-start{align-items:flex-start!important}.align-items-end{align-items:flex-end!important}.align-items-center{align-items:center!important}.align-items-baseline{align-items:baseline!important}.align-items-stretch{align-items:stretch!important}.align-content-start{align-content:flex-start!important}.align-content-end{align-content:flex-end!important}.align-content-center{align-content:center!important}.align-content-between{align-content:space-between!important}.align-content-around{align-content:space-around!important}.align-content-stretch{align-content:stretch!important}.align-self-auto{align-self:auto!important}.align-self-start{align-self:flex-start!important}.align-self-end{align-self:flex-end!important}.align-self-center{align-self:center!important}.align-self-baseline{align-self:baseline!important}.align-self-stretch{align-self:stretch!important}.order-first{order:-1!important}.order-0{order:0!important}.order-1{order:1!important}.order-2{order:2!important}.order-3{order:3!important}.order-4{order:4!important}.order-5{order:5!important}.order-last{order:6!important}.m-0{margin:0!important}.m-1{margin:.25rem!important}.m-2{margin:.5rem!important}.m-3{margin:1rem!important}.m-4{margin:1.5rem!important}.m-5{margin:3rem!important}.m-auto{margin:auto!important}.mx-0{margin-right:0!important;margin-left:0!important}.mx-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-3{margin-right:1rem!important;margin-left:1rem!important}.mx-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-5{margin-right:3rem!important;margin-left:3rem!important}.mx-auto{margin-right:auto!important;margin-left:auto!important}.my-0{margin-top:0!important;margin-bottom:0!important}.my-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-0{margin-top:0!important}.mt-1{margin-top:.25rem!important}.mt-2{margin-top:.5rem!important}.mt-3{margin-top:1rem!important}.mt-4{margin-top:1.5rem!important}.mt-5{margin-top:3rem!important}.mt-auto{margin-top:auto!important}.me-0{margin-right:0!important}.me-1{margin-right:.25rem!important}.me-2{margin-right:.5rem!important}.me-3{margin-right:1rem!important}.me-4{margin-right:1.5rem!important}.me-5{margin-right:3rem!important}.me-auto{margin-right:auto!important}.mb-0{margin-bottom:0!important}.mb-1{margin-bottom:.25rem!important}.mb-2{margin-bottom:.5rem!important}.mb-3{margin-bottom:1rem!important}.mb-4{margin-bottom:1.5rem!important}.mb-5{margin-bottom:3rem!important}.mb-auto{margin-bottom:auto!important}.ms-0{margin-left:0!important}.ms-1{margin-left:.25rem!important}.ms-2{margin-left:.5rem!important}.ms-3{margin-left:1rem!important}.ms-4{margin-left:1.5rem!important}.ms-5{margin-left:3rem!important}.ms-auto{margin-left:auto!important}.p-0{padding:0!important}.p-1{padding:.25rem!important}.p-2{padding:.5rem!important}.p-3{padding:1rem!important}.p-4{padding:1.5rem!important}.p-5{padding:3rem!important}.px-0{padding-right:0!important;padding-left:0!important}.px-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-3{padding-right:1rem!important;padding-left:1rem!important}.px-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-5{padding-right:3rem!important;padding-left:3rem!important}.py-0{padding-top:0!important;padding-bottom:0!important}.py-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-0{padding-top:0!important}.pt-1{padding-top:.25rem!important}.pt-2{padding-top:.5rem!important}.pt-3{padding-top:1rem!important}.pt-4{padding-top:1.5rem!important}.pt-5{padding-top:3rem!important}.pe-0{padding-right:0!important}.pe-1{padding-right:.25rem!important}.pe-2{padding-right:.5rem!important}.pe-3{padding-right:1rem!important}.pe-4{padding-right:1.5rem!important}.pe-5{padding-right:3rem!important}.pb-0{padding-bottom:0!important}.pb-1{padding-bottom:.25rem!important}.pb-2{padding-bottom:.5rem!important}.pb-3{padding-bottom:1rem!important}.pb-4{padding-bottom:1.5rem!important}.pb-5{padding-bottom:3rem!important}.ps-0{padding-left:0!important}.ps-1{padding-left:.25rem!important}.ps-2{padding-left:.5rem!important}.ps-3{padding-left:1rem!important}.ps-4{padding-left:1.5rem!important}.ps-5{padding-left:3rem!important}.font-monospace{font-family:var(--bs-font-monospace)!important}.fs-1{font-size:calc(1.375rem + 1.5vw)!important}.fs-2{font-size:calc(1.325rem + .9vw)!important}.fs-3{font-size:calc(1.3rem + .6vw)!important}.fs-4{font-size:calc(1.275rem + .3vw)!important}.fs-5{font-size:1.25rem!important}.fs-6{font-size:1rem!important}.fst-italic{font-style:italic!important}.fst-normal{font-style:normal!important}.fw-light{font-weight:300!important}.fw-lighter{font-weight:lighter!important}.fw-normal{font-weight:400!important}.fw-bold{font-weight:700!important}.fw-bolder{font-weight:bolder!important}.lh-1{line-height:1!important}.lh-sm{line-height:1.25!important}.lh-base{line-height:1.5!important}.lh-lg{line-height:2!important}.text-start{text-align:left!important}.text-end{text-align:right!important}.text-center{text-align:center!important}.text-decoration-none{text-decoration:none!important}.text-decoration-underline{text-decoration:underline!important}.text-decoration-line-through{text-decoration:line-through!important}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-break{word-wrap:break-word!important;word-break:break-word!important}.text-primary{--bs-text-opacity:1;color:rgba(var(--bs-primary-rgb),var(--bs-text-opacity))!important}.text-secondary{--bs-text-opacity:1;color:rgba(var(--bs-secondary-rgb),var(--bs-text-opacity))!important}.text-success{--bs-text-opacity:1;color:rgba(var(--bs-success-rgb),var(--bs-text-opacity))!important}.text-info{--bs-text-opacity:1;color:rgba(var(--bs-info-rgb),var(--bs-text-opacity))!important}.text-warning{--bs-text-opacity:1;color:rgba(var(--bs-warning-rgb),var(--bs-text-opacity))!important}.text-danger{--bs-text-opacity:1;color:rgba(var(--bs-danger-rgb),var(--bs-text-opacity))!important}.text-light{--bs-text-opacity:1;color:rgba(var(--bs-light-rgb),var(--bs-text-opacity))!important}.text-dark{--bs-text-opacity:1;color:rgba(var(--bs-dark-rgb),var(--bs-text-opacity))!important}.text-black{--bs-text-opacity:1;color:rgba(var(--bs-black-rgb),var(--bs-text-opacity))!important}.text-white{--bs-text-opacity:1;color:rgba(var(--bs-white-rgb),var(--bs-text-opacity))!important}.text-body{--bs-text-opacity:1;color:rgba(var(--bs-body-rgb),var(--bs-text-opacity))!important}.text-muted{--bs-text-opacity:1;color:#6c757d!important}.text-black-50{--bs-text-opacity:1;color:rgba(0,0,0,.5)!important}.text-white-50{--bs-text-opacity:1;color:rgba(255,255,255,.5)!important}.text-reset{--bs-text-opacity:1;color:inherit!important}.text-opacity-25{--bs-text-opacity:0.25}.text-opacity-50{--bs-text-opacity:0.5}.text-opacity-75{--bs-text-opacity:0.75}.text-opacity-100{--bs-text-opacity:1}.bg-primary{--bs-bg-opacity:1;background-color:rgba(var(--bs-primary-rgb),var(--bs-bg-opacity))!important}.bg-secondary{--bs-bg-opacity:1;background-color:rgba(var(--bs-secondary-rgb),var(--bs-bg-opacity))!important}.bg-success{--bs-bg-opacity:1;background-color:rgba(var(--bs-success-rgb),var(--bs-bg-opacity))!important}.bg-info{--bs-bg-opacity:1;background-color:rgba(var(--bs-info-rgb),var(--bs-bg-opacity))!important}.bg-warning{--bs-bg-opacity:1;background-color:rgba(var(--bs-warning-rgb),var(--bs-bg-opacity))!important}.bg-danger{--bs-bg-opacity:1;background-color:rgba(var(--bs-danger-rgb),var(--bs-bg-opacity))!important}.bg-light{--bs-bg-opacity:1;background-color:rgba(var(--bs-light-rgb),var(--bs-bg-opacity))!important}.bg-dark{--bs-bg-opacity:1;background-color:rgba(var(--bs-dark-rgb),var(--bs-bg-opacity))!important}.bg-black{--bs-bg-opacity:1;background-color:rgba(var(--bs-black-rgb),var(--bs-bg-opacity))!important}.bg-white{--bs-bg-opacity:1;background-color:rgba(var(--bs-white-rgb),var(--bs-bg-opacity))!important}.bg-body{--bs-bg-opacity:1;background-color:rgba(var(--bs-body-rgb),var(--bs-bg-opacity))!important}.bg-transparent{--bs-bg-opacity:1;background-color:transparent!important}.bg-opacity-10{--bs-bg-opacity:0.1}.bg-opacity-25{--bs-bg-opacity:0.25}.bg-opacity-50{--bs-bg-opacity:0.5}.bg-opacity-75{--bs-bg-opacity:0.75}.bg-opacity-100{--bs-bg-opacity:1}.bg-gradient{background-image:var(--bs-gradient)!important}.user-select-all{-webkit-user-select:all!important;-moz-user-select:all!important;user-select:all!important}.user-select-auto{-webkit-user-select:auto!important;-moz-user-select:auto!important;user-select:auto!important}.user-select-none{-webkit-user-select:none!important;-moz-user-select:none!important;user-select:none!important}.pe-none{pointer-events:none!important}.pe-auto{pointer-events:auto!important}.rounded{border-radius:.25rem!important}.rounded-0{border-radius:0!important}.rounded-1{border-radius:.2rem!important}.rounded-2{border-radius:.25rem!important}.rounded-3{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-end{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-start{border-bottom-left-radius:.25rem!important;border-top-left-radius:.25rem!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media (min-width:576px){.float-sm-start{float:left!important}.float-sm-end{float:right!important}.float-sm-none{float:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-grid{display:grid!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:flex!important}.d-sm-inline-flex{display:inline-flex!important}.d-sm-none{display:none!important}.flex-sm-fill{flex:1 1 auto!important}.flex-sm-row{flex-direction:row!important}.flex-sm-column{flex-direction:column!important}.flex-sm-row-reverse{flex-direction:row-reverse!important}.flex-sm-column-reverse{flex-direction:column-reverse!important}.flex-sm-grow-0{flex-grow:0!important}.flex-sm-grow-1{flex-grow:1!important}.flex-sm-shrink-0{flex-shrink:0!important}.flex-sm-shrink-1{flex-shrink:1!important}.flex-sm-wrap{flex-wrap:wrap!important}.flex-sm-nowrap{flex-wrap:nowrap!important}.flex-sm-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-sm-0{gap:0!important}.gap-sm-1{gap:.25rem!important}.gap-sm-2{gap:.5rem!important}.gap-sm-3{gap:1rem!important}.gap-sm-4{gap:1.5rem!important}.gap-sm-5{gap:3rem!important}.justify-content-sm-start{justify-content:flex-start!important}.justify-content-sm-end{justify-content:flex-end!important}.justify-content-sm-center{justify-content:center!important}.justify-content-sm-between{justify-content:space-between!important}.justify-content-sm-around{justify-content:space-around!important}.justify-content-sm-evenly{justify-content:space-evenly!important}.align-items-sm-start{align-items:flex-start!important}.align-items-sm-end{align-items:flex-end!important}.align-items-sm-center{align-items:center!important}.align-items-sm-baseline{align-items:baseline!important}.align-items-sm-stretch{align-items:stretch!important}.align-content-sm-start{align-content:flex-start!important}.align-content-sm-end{align-content:flex-end!important}.align-content-sm-center{align-content:center!important}.align-content-sm-between{align-content:space-between!important}.align-content-sm-around{align-content:space-around!important}.align-content-sm-stretch{align-content:stretch!important}.align-self-sm-auto{align-self:auto!important}.align-self-sm-start{align-self:flex-start!important}.align-self-sm-end{align-self:flex-end!important}.align-self-sm-center{align-self:center!important}.align-self-sm-baseline{align-self:baseline!important}.align-self-sm-stretch{align-self:stretch!important}.order-sm-first{order:-1!important}.order-sm-0{order:0!important}.order-sm-1{order:1!important}.order-sm-2{order:2!important}.order-sm-3{order:3!important}.order-sm-4{order:4!important}.order-sm-5{order:5!important}.order-sm-last{order:6!important}.m-sm-0{margin:0!important}.m-sm-1{margin:.25rem!important}.m-sm-2{margin:.5rem!important}.m-sm-3{margin:1rem!important}.m-sm-4{margin:1.5rem!important}.m-sm-5{margin:3rem!important}.m-sm-auto{margin:auto!important}.mx-sm-0{margin-right:0!important;margin-left:0!important}.mx-sm-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-sm-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-sm-3{margin-right:1rem!important;margin-left:1rem!important}.mx-sm-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-sm-5{margin-right:3rem!important;margin-left:3rem!important}.mx-sm-auto{margin-right:auto!important;margin-left:auto!important}.my-sm-0{margin-top:0!important;margin-bottom:0!important}.my-sm-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-sm-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-sm-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-sm-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-sm-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-sm-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-sm-0{margin-top:0!important}.mt-sm-1{margin-top:.25rem!important}.mt-sm-2{margin-top:.5rem!important}.mt-sm-3{margin-top:1rem!important}.mt-sm-4{margin-top:1.5rem!important}.mt-sm-5{margin-top:3rem!important}.mt-sm-auto{margin-top:auto!important}.me-sm-0{margin-right:0!important}.me-sm-1{margin-right:.25rem!important}.me-sm-2{margin-right:.5rem!important}.me-sm-3{margin-right:1rem!important}.me-sm-4{margin-right:1.5rem!important}.me-sm-5{margin-right:3rem!important}.me-sm-auto{margin-right:auto!important}.mb-sm-0{margin-bottom:0!important}.mb-sm-1{margin-bottom:.25rem!important}.mb-sm-2{margin-bottom:.5rem!important}.mb-sm-3{margin-bottom:1rem!important}.mb-sm-4{margin-bottom:1.5rem!important}.mb-sm-5{margin-bottom:3rem!important}.mb-sm-auto{margin-bottom:auto!important}.ms-sm-0{margin-left:0!important}.ms-sm-1{margin-left:.25rem!important}.ms-sm-2{margin-left:.5rem!important}.ms-sm-3{margin-left:1rem!important}.ms-sm-4{margin-left:1.5rem!important}.ms-sm-5{margin-left:3rem!important}.ms-sm-auto{margin-left:auto!important}.p-sm-0{padding:0!important}.p-sm-1{padding:.25rem!important}.p-sm-2{padding:.5rem!important}.p-sm-3{padding:1rem!important}.p-sm-4{padding:1.5rem!important}.p-sm-5{padding:3rem!important}.px-sm-0{padding-right:0!important;padding-left:0!important}.px-sm-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-sm-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-sm-3{padding-right:1rem!important;padding-left:1rem!important}.px-sm-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-sm-5{padding-right:3rem!important;padding-left:3rem!important}.py-sm-0{padding-top:0!important;padding-bottom:0!important}.py-sm-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-sm-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-sm-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-sm-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-sm-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-sm-0{padding-top:0!important}.pt-sm-1{padding-top:.25rem!important}.pt-sm-2{padding-top:.5rem!important}.pt-sm-3{padding-top:1rem!important}.pt-sm-4{padding-top:1.5rem!important}.pt-sm-5{padding-top:3rem!important}.pe-sm-0{padding-right:0!important}.pe-sm-1{padding-right:.25rem!important}.pe-sm-2{padding-right:.5rem!important}.pe-sm-3{padding-right:1rem!important}.pe-sm-4{padding-right:1.5rem!important}.pe-sm-5{padding-right:3rem!important}.pb-sm-0{padding-bottom:0!important}.pb-sm-1{padding-bottom:.25rem!important}.pb-sm-2{padding-bottom:.5rem!important}.pb-sm-3{padding-bottom:1rem!important}.pb-sm-4{padding-bottom:1.5rem!important}.pb-sm-5{padding-bottom:3rem!important}.ps-sm-0{padding-left:0!important}.ps-sm-1{padding-left:.25rem!important}.ps-sm-2{padding-left:.5rem!important}.ps-sm-3{padding-left:1rem!important}.ps-sm-4{padding-left:1.5rem!important}.ps-sm-5{padding-left:3rem!important}.text-sm-start{text-align:left!important}.text-sm-end{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.float-md-start{float:left!important}.float-md-end{float:right!important}.float-md-none{float:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-grid{display:grid!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:flex!important}.d-md-inline-flex{display:inline-flex!important}.d-md-none{display:none!important}.flex-md-fill{flex:1 1 auto!important}.flex-md-row{flex-direction:row!important}.flex-md-column{flex-direction:column!important}.flex-md-row-reverse{flex-direction:row-reverse!important}.flex-md-column-reverse{flex-direction:column-reverse!important}.flex-md-grow-0{flex-grow:0!important}.flex-md-grow-1{flex-grow:1!important}.flex-md-shrink-0{flex-shrink:0!important}.flex-md-shrink-1{flex-shrink:1!important}.flex-md-wrap{flex-wrap:wrap!important}.flex-md-nowrap{flex-wrap:nowrap!important}.flex-md-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-md-0{gap:0!important}.gap-md-1{gap:.25rem!important}.gap-md-2{gap:.5rem!important}.gap-md-3{gap:1rem!important}.gap-md-4{gap:1.5rem!important}.gap-md-5{gap:3rem!important}.justify-content-md-start{justify-content:flex-start!important}.justify-content-md-end{justify-content:flex-end!important}.justify-content-md-center{justify-content:center!important}.justify-content-md-between{justify-content:space-between!important}.justify-content-md-around{justify-content:space-around!important}.justify-content-md-evenly{justify-content:space-evenly!important}.align-items-md-start{align-items:flex-start!important}.align-items-md-end{align-items:flex-end!important}.align-items-md-center{align-items:center!important}.align-items-md-baseline{align-items:baseline!important}.align-items-md-stretch{align-items:stretch!important}.align-content-md-start{align-content:flex-start!important}.align-content-md-end{align-content:flex-end!important}.align-content-md-center{align-content:center!important}.align-content-md-between{align-content:space-between!important}.align-content-md-around{align-content:space-around!important}.align-content-md-stretch{align-content:stretch!important}.align-self-md-auto{align-self:auto!important}.align-self-md-start{align-self:flex-start!important}.align-self-md-end{align-self:flex-end!important}.align-self-md-center{align-self:center!important}.align-self-md-baseline{align-self:baseline!important}.align-self-md-stretch{align-self:stretch!important}.order-md-first{order:-1!important}.order-md-0{order:0!important}.order-md-1{order:1!important}.order-md-2{order:2!important}.order-md-3{order:3!important}.order-md-4{order:4!important}.order-md-5{order:5!important}.order-md-last{order:6!important}.m-md-0{margin:0!important}.m-md-1{margin:.25rem!important}.m-md-2{margin:.5rem!important}.m-md-3{margin:1rem!important}.m-md-4{margin:1.5rem!important}.m-md-5{margin:3rem!important}.m-md-auto{margin:auto!important}.mx-md-0{margin-right:0!important;margin-left:0!important}.mx-md-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-md-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-md-3{margin-right:1rem!important;margin-left:1rem!important}.mx-md-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-md-5{margin-right:3rem!important;margin-left:3rem!important}.mx-md-auto{margin-right:auto!important;margin-left:auto!important}.my-md-0{margin-top:0!important;margin-bottom:0!important}.my-md-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-md-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-md-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-md-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-md-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-md-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-md-0{margin-top:0!important}.mt-md-1{margin-top:.25rem!important}.mt-md-2{margin-top:.5rem!important}.mt-md-3{margin-top:1rem!important}.mt-md-4{margin-top:1.5rem!important}.mt-md-5{margin-top:3rem!important}.mt-md-auto{margin-top:auto!important}.me-md-0{margin-right:0!important}.me-md-1{margin-right:.25rem!important}.me-md-2{margin-right:.5rem!important}.me-md-3{margin-right:1rem!important}.me-md-4{margin-right:1.5rem!important}.me-md-5{margin-right:3rem!important}.me-md-auto{margin-right:auto!important}.mb-md-0{margin-bottom:0!important}.mb-md-1{margin-bottom:.25rem!important}.mb-md-2{margin-bottom:.5rem!important}.mb-md-3{margin-bottom:1rem!important}.mb-md-4{margin-bottom:1.5rem!important}.mb-md-5{margin-bottom:3rem!important}.mb-md-auto{margin-bottom:auto!important}.ms-md-0{margin-left:0!important}.ms-md-1{margin-left:.25rem!important}.ms-md-2{margin-left:.5rem!important}.ms-md-3{margin-left:1rem!important}.ms-md-4{margin-left:1.5rem!important}.ms-md-5{margin-left:3rem!important}.ms-md-auto{margin-left:auto!important}.p-md-0{padding:0!important}.p-md-1{padding:.25rem!important}.p-md-2{padding:.5rem!important}.p-md-3{padding:1rem!important}.p-md-4{padding:1.5rem!important}.p-md-5{padding:3rem!important}.px-md-0{padding-right:0!important;padding-left:0!important}.px-md-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-md-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-md-3{padding-right:1rem!important;padding-left:1rem!important}.px-md-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-md-5{padding-right:3rem!important;padding-left:3rem!important}.py-md-0{padding-top:0!important;padding-bottom:0!important}.py-md-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-md-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-md-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-md-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-md-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-md-0{padding-top:0!important}.pt-md-1{padding-top:.25rem!important}.pt-md-2{padding-top:.5rem!important}.pt-md-3{padding-top:1rem!important}.pt-md-4{padding-top:1.5rem!important}.pt-md-5{padding-top:3rem!important}.pe-md-0{padding-right:0!important}.pe-md-1{padding-right:.25rem!important}.pe-md-2{padding-right:.5rem!important}.pe-md-3{padding-right:1rem!important}.pe-md-4{padding-right:1.5rem!important}.pe-md-5{padding-right:3rem!important}.pb-md-0{padding-bottom:0!important}.pb-md-1{padding-bottom:.25rem!important}.pb-md-2{padding-bottom:.5rem!important}.pb-md-3{padding-bottom:1rem!important}.pb-md-4{padding-bottom:1.5rem!important}.pb-md-5{padding-bottom:3rem!important}.ps-md-0{padding-left:0!important}.ps-md-1{padding-left:.25rem!important}.ps-md-2{padding-left:.5rem!important}.ps-md-3{padding-left:1rem!important}.ps-md-4{padding-left:1.5rem!important}.ps-md-5{padding-left:3rem!important}.text-md-start{text-align:left!important}.text-md-end{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.float-lg-start{float:left!important}.float-lg-end{float:right!important}.float-lg-none{float:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-grid{display:grid!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:flex!important}.d-lg-inline-flex{display:inline-flex!important}.d-lg-none{display:none!important}.flex-lg-fill{flex:1 1 auto!important}.flex-lg-row{flex-direction:row!important}.flex-lg-column{flex-direction:column!important}.flex-lg-row-reverse{flex-direction:row-reverse!important}.flex-lg-column-reverse{flex-direction:column-reverse!important}.flex-lg-grow-0{flex-grow:0!important}.flex-lg-grow-1{flex-grow:1!important}.flex-lg-shrink-0{flex-shrink:0!important}.flex-lg-shrink-1{flex-shrink:1!important}.flex-lg-wrap{flex-wrap:wrap!important}.flex-lg-nowrap{flex-wrap:nowrap!important}.flex-lg-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-lg-0{gap:0!important}.gap-lg-1{gap:.25rem!important}.gap-lg-2{gap:.5rem!important}.gap-lg-3{gap:1rem!important}.gap-lg-4{gap:1.5rem!important}.gap-lg-5{gap:3rem!important}.justify-content-lg-start{justify-content:flex-start!important}.justify-content-lg-end{justify-content:flex-end!important}.justify-content-lg-center{justify-content:center!important}.justify-content-lg-between{justify-content:space-between!important}.justify-content-lg-around{justify-content:space-around!important}.justify-content-lg-evenly{justify-content:space-evenly!important}.align-items-lg-start{align-items:flex-start!important}.align-items-lg-end{align-items:flex-end!important}.align-items-lg-center{align-items:center!important}.align-items-lg-baseline{align-items:baseline!important}.align-items-lg-stretch{align-items:stretch!important}.align-content-lg-start{align-content:flex-start!important}.align-content-lg-end{align-content:flex-end!important}.align-content-lg-center{align-content:center!important}.align-content-lg-between{align-content:space-between!important}.align-content-lg-around{align-content:space-around!important}.align-content-lg-stretch{align-content:stretch!important}.align-self-lg-auto{align-self:auto!important}.align-self-lg-start{align-self:flex-start!important}.align-self-lg-end{align-self:flex-end!important}.align-self-lg-center{align-self:center!important}.align-self-lg-baseline{align-self:baseline!important}.align-self-lg-stretch{align-self:stretch!important}.order-lg-first{order:-1!important}.order-lg-0{order:0!important}.order-lg-1{order:1!important}.order-lg-2{order:2!important}.order-lg-3{order:3!important}.order-lg-4{order:4!important}.order-lg-5{order:5!important}.order-lg-last{order:6!important}.m-lg-0{margin:0!important}.m-lg-1{margin:.25rem!important}.m-lg-2{margin:.5rem!important}.m-lg-3{margin:1rem!important}.m-lg-4{margin:1.5rem!important}.m-lg-5{margin:3rem!important}.m-lg-auto{margin:auto!important}.mx-lg-0{margin-right:0!important;margin-left:0!important}.mx-lg-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-lg-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-lg-3{margin-right:1rem!important;margin-left:1rem!important}.mx-lg-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-lg-5{margin-right:3rem!important;margin-left:3rem!important}.mx-lg-auto{margin-right:auto!important;margin-left:auto!important}.my-lg-0{margin-top:0!important;margin-bottom:0!important}.my-lg-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-lg-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-lg-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-lg-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-lg-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-lg-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-lg-0{margin-top:0!important}.mt-lg-1{margin-top:.25rem!important}.mt-lg-2{margin-top:.5rem!important}.mt-lg-3{margin-top:1rem!important}.mt-lg-4{margin-top:1.5rem!important}.mt-lg-5{margin-top:3rem!important}.mt-lg-auto{margin-top:auto!important}.me-lg-0{margin-right:0!important}.me-lg-1{margin-right:.25rem!important}.me-lg-2{margin-right:.5rem!important}.me-lg-3{margin-right:1rem!important}.me-lg-4{margin-right:1.5rem!important}.me-lg-5{margin-right:3rem!important}.me-lg-auto{margin-right:auto!important}.mb-lg-0{margin-bottom:0!important}.mb-lg-1{margin-bottom:.25rem!important}.mb-lg-2{margin-bottom:.5rem!important}.mb-lg-3{margin-bottom:1rem!important}.mb-lg-4{margin-bottom:1.5rem!important}.mb-lg-5{margin-bottom:3rem!important}.mb-lg-auto{margin-bottom:auto!important}.ms-lg-0{margin-left:0!important}.ms-lg-1{margin-left:.25rem!important}.ms-lg-2{margin-left:.5rem!important}.ms-lg-3{margin-left:1rem!important}.ms-lg-4{margin-left:1.5rem!important}.ms-lg-5{margin-left:3rem!important}.ms-lg-auto{margin-left:auto!important}.p-lg-0{padding:0!important}.p-lg-1{padding:.25rem!important}.p-lg-2{padding:.5rem!important}.p-lg-3{padding:1rem!important}.p-lg-4{padding:1.5rem!important}.p-lg-5{padding:3rem!important}.px-lg-0{padding-right:0!important;padding-left:0!important}.px-lg-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-lg-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-lg-3{padding-right:1rem!important;padding-left:1rem!important}.px-lg-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-lg-5{padding-right:3rem!important;padding-left:3rem!important}.py-lg-0{padding-top:0!important;padding-bottom:0!important}.py-lg-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-lg-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-lg-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-lg-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-lg-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-lg-0{padding-top:0!important}.pt-lg-1{padding-top:.25rem!important}.pt-lg-2{padding-top:.5rem!important}.pt-lg-3{padding-top:1rem!important}.pt-lg-4{padding-top:1.5rem!important}.pt-lg-5{padding-top:3rem!important}.pe-lg-0{padding-right:0!important}.pe-lg-1{padding-right:.25rem!important}.pe-lg-2{padding-right:.5rem!important}.pe-lg-3{padding-right:1rem!important}.pe-lg-4{padding-right:1.5rem!important}.pe-lg-5{padding-right:3rem!important}.pb-lg-0{padding-bottom:0!important}.pb-lg-1{padding-bottom:.25rem!important}.pb-lg-2{padding-bottom:.5rem!important}.pb-lg-3{padding-bottom:1rem!important}.pb-lg-4{padding-bottom:1.5rem!important}.pb-lg-5{padding-bottom:3rem!important}.ps-lg-0{padding-left:0!important}.ps-lg-1{padding-left:.25rem!important}.ps-lg-2{padding-left:.5rem!important}.ps-lg-3{padding-left:1rem!important}.ps-lg-4{padding-left:1.5rem!important}.ps-lg-5{padding-left:3rem!important}.text-lg-start{text-align:left!important}.text-lg-end{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.float-xl-start{float:left!important}.float-xl-end{float:right!important}.float-xl-none{float:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-grid{display:grid!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:flex!important}.d-xl-inline-flex{display:inline-flex!important}.d-xl-none{display:none!important}.flex-xl-fill{flex:1 1 auto!important}.flex-xl-row{flex-direction:row!important}.flex-xl-column{flex-direction:column!important}.flex-xl-row-reverse{flex-direction:row-reverse!important}.flex-xl-column-reverse{flex-direction:column-reverse!important}.flex-xl-grow-0{flex-grow:0!important}.flex-xl-grow-1{flex-grow:1!important}.flex-xl-shrink-0{flex-shrink:0!important}.flex-xl-shrink-1{flex-shrink:1!important}.flex-xl-wrap{flex-wrap:wrap!important}.flex-xl-nowrap{flex-wrap:nowrap!important}.flex-xl-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-xl-0{gap:0!important}.gap-xl-1{gap:.25rem!important}.gap-xl-2{gap:.5rem!important}.gap-xl-3{gap:1rem!important}.gap-xl-4{gap:1.5rem!important}.gap-xl-5{gap:3rem!important}.justify-content-xl-start{justify-content:flex-start!important}.justify-content-xl-end{justify-content:flex-end!important}.justify-content-xl-center{justify-content:center!important}.justify-content-xl-between{justify-content:space-between!important}.justify-content-xl-around{justify-content:space-around!important}.justify-content-xl-evenly{justify-content:space-evenly!important}.align-items-xl-start{align-items:flex-start!important}.align-items-xl-end{align-items:flex-end!important}.align-items-xl-center{align-items:center!important}.align-items-xl-baseline{align-items:baseline!important}.align-items-xl-stretch{align-items:stretch!important}.align-content-xl-start{align-content:flex-start!important}.align-content-xl-end{align-content:flex-end!important}.align-content-xl-center{align-content:center!important}.align-content-xl-between{align-content:space-between!important}.align-content-xl-around{align-content:space-around!important}.align-content-xl-stretch{align-content:stretch!important}.align-self-xl-auto{align-self:auto!important}.align-self-xl-start{align-self:flex-start!important}.align-self-xl-end{align-self:flex-end!important}.align-self-xl-center{align-self:center!important}.align-self-xl-baseline{align-self:baseline!important}.align-self-xl-stretch{align-self:stretch!important}.order-xl-first{order:-1!important}.order-xl-0{order:0!important}.order-xl-1{order:1!important}.order-xl-2{order:2!important}.order-xl-3{order:3!important}.order-xl-4{order:4!important}.order-xl-5{order:5!important}.order-xl-last{order:6!important}.m-xl-0{margin:0!important}.m-xl-1{margin:.25rem!important}.m-xl-2{margin:.5rem!important}.m-xl-3{margin:1rem!important}.m-xl-4{margin:1.5rem!important}.m-xl-5{margin:3rem!important}.m-xl-auto{margin:auto!important}.mx-xl-0{margin-right:0!important;margin-left:0!important}.mx-xl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xl-auto{margin-right:auto!important;margin-left:auto!important}.my-xl-0{margin-top:0!important;margin-bottom:0!important}.my-xl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xl-0{margin-top:0!important}.mt-xl-1{margin-top:.25rem!important}.mt-xl-2{margin-top:.5rem!important}.mt-xl-3{margin-top:1rem!important}.mt-xl-4{margin-top:1.5rem!important}.mt-xl-5{margin-top:3rem!important}.mt-xl-auto{margin-top:auto!important}.me-xl-0{margin-right:0!important}.me-xl-1{margin-right:.25rem!important}.me-xl-2{margin-right:.5rem!important}.me-xl-3{margin-right:1rem!important}.me-xl-4{margin-right:1.5rem!important}.me-xl-5{margin-right:3rem!important}.me-xl-auto{margin-right:auto!important}.mb-xl-0{margin-bottom:0!important}.mb-xl-1{margin-bottom:.25rem!important}.mb-xl-2{margin-bottom:.5rem!important}.mb-xl-3{margin-bottom:1rem!important}.mb-xl-4{margin-bottom:1.5rem!important}.mb-xl-5{margin-bottom:3rem!important}.mb-xl-auto{margin-bottom:auto!important}.ms-xl-0{margin-left:0!important}.ms-xl-1{margin-left:.25rem!important}.ms-xl-2{margin-left:.5rem!important}.ms-xl-3{margin-left:1rem!important}.ms-xl-4{margin-left:1.5rem!important}.ms-xl-5{margin-left:3rem!important}.ms-xl-auto{margin-left:auto!important}.p-xl-0{padding:0!important}.p-xl-1{padding:.25rem!important}.p-xl-2{padding:.5rem!important}.p-xl-3{padding:1rem!important}.p-xl-4{padding:1.5rem!important}.p-xl-5{padding:3rem!important}.px-xl-0{padding-right:0!important;padding-left:0!important}.px-xl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xl-0{padding-top:0!important;padding-bottom:0!important}.py-xl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xl-0{padding-top:0!important}.pt-xl-1{padding-top:.25rem!important}.pt-xl-2{padding-top:.5rem!important}.pt-xl-3{padding-top:1rem!important}.pt-xl-4{padding-top:1.5rem!important}.pt-xl-5{padding-top:3rem!important}.pe-xl-0{padding-right:0!important}.pe-xl-1{padding-right:.25rem!important}.pe-xl-2{padding-right:.5rem!important}.pe-xl-3{padding-right:1rem!important}.pe-xl-4{padding-right:1.5rem!important}.pe-xl-5{padding-right:3rem!important}.pb-xl-0{padding-bottom:0!important}.pb-xl-1{padding-bottom:.25rem!important}.pb-xl-2{padding-bottom:.5rem!important}.pb-xl-3{padding-bottom:1rem!important}.pb-xl-4{padding-bottom:1.5rem!important}.pb-xl-5{padding-bottom:3rem!important}.ps-xl-0{padding-left:0!important}.ps-xl-1{padding-left:.25rem!important}.ps-xl-2{padding-left:.5rem!important}.ps-xl-3{padding-left:1rem!important}.ps-xl-4{padding-left:1.5rem!important}.ps-xl-5{padding-left:3rem!important}.text-xl-start{text-align:left!important}.text-xl-end{text-align:right!important}.text-xl-center{text-align:center!important}}@media (min-width:1400px){.float-xxl-start{float:left!important}.float-xxl-end{float:right!important}.float-xxl-none{float:none!important}.d-xxl-inline{display:inline!important}.d-xxl-inline-block{display:inline-block!important}.d-xxl-block{display:block!important}.d-xxl-grid{display:grid!important}.d-xxl-table{display:table!important}.d-xxl-table-row{display:table-row!important}.d-xxl-table-cell{display:table-cell!important}.d-xxl-flex{display:flex!important}.d-xxl-inline-flex{display:inline-flex!important}.d-xxl-none{display:none!important}.flex-xxl-fill{flex:1 1 auto!important}.flex-xxl-row{flex-direction:row!important}.flex-xxl-column{flex-direction:column!important}.flex-xxl-row-reverse{flex-direction:row-reverse!important}.flex-xxl-column-reverse{flex-direction:column-reverse!important}.flex-xxl-grow-0{flex-grow:0!important}.flex-xxl-grow-1{flex-grow:1!important}.flex-xxl-shrink-0{flex-shrink:0!important}.flex-xxl-shrink-1{flex-shrink:1!important}.flex-xxl-wrap{flex-wrap:wrap!important}.flex-xxl-nowrap{flex-wrap:nowrap!important}.flex-xxl-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-xxl-0{gap:0!important}.gap-xxl-1{gap:.25rem!important}.gap-xxl-2{gap:.5rem!important}.gap-xxl-3{gap:1rem!important}.gap-xxl-4{gap:1.5rem!important}.gap-xxl-5{gap:3rem!important}.justify-content-xxl-start{justify-content:flex-start!important}.justify-content-xxl-end{justify-content:flex-end!important}.justify-content-xxl-center{justify-content:center!important}.justify-content-xxl-between{justify-content:space-between!important}.justify-content-xxl-around{justify-content:space-around!important}.justify-content-xxl-evenly{justify-content:space-evenly!important}.align-items-xxl-start{align-items:flex-start!important}.align-items-xxl-end{align-items:flex-end!important}.align-items-xxl-center{align-items:center!important}.align-items-xxl-baseline{align-items:baseline!important}.align-items-xxl-stretch{align-items:stretch!important}.align-content-xxl-start{align-content:flex-start!important}.align-content-xxl-end{align-content:flex-end!important}.align-content-xxl-center{align-content:center!important}.align-content-xxl-between{align-content:space-between!important}.align-content-xxl-around{align-content:space-around!important}.align-content-xxl-stretch{align-content:stretch!important}.align-self-xxl-auto{align-self:auto!important}.align-self-xxl-start{align-self:flex-start!important}.align-self-xxl-end{align-self:flex-end!important}.align-self-xxl-center{align-self:center!important}.align-self-xxl-baseline{align-self:baseline!important}.align-self-xxl-stretch{align-self:stretch!important}.order-xxl-first{order:-1!important}.order-xxl-0{order:0!important}.order-xxl-1{order:1!important}.order-xxl-2{order:2!important}.order-xxl-3{order:3!important}.order-xxl-4{order:4!important}.order-xxl-5{order:5!important}.order-xxl-last{order:6!important}.m-xxl-0{margin:0!important}.m-xxl-1{margin:.25rem!important}.m-xxl-2{margin:.5rem!important}.m-xxl-3{margin:1rem!important}.m-xxl-4{margin:1.5rem!important}.m-xxl-5{margin:3rem!important}.m-xxl-auto{margin:auto!important}.mx-xxl-0{margin-right:0!important;margin-left:0!important}.mx-xxl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xxl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xxl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xxl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xxl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xxl-auto{margin-right:auto!important;margin-left:auto!important}.my-xxl-0{margin-top:0!important;margin-bottom:0!important}.my-xxl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xxl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xxl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xxl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xxl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xxl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xxl-0{margin-top:0!important}.mt-xxl-1{margin-top:.25rem!important}.mt-xxl-2{margin-top:.5rem!important}.mt-xxl-3{margin-top:1rem!important}.mt-xxl-4{margin-top:1.5rem!important}.mt-xxl-5{margin-top:3rem!important}.mt-xxl-auto{margin-top:auto!important}.me-xxl-0{margin-right:0!important}.me-xxl-1{margin-right:.25rem!important}.me-xxl-2{margin-right:.5rem!important}.me-xxl-3{margin-right:1rem!important}.me-xxl-4{margin-right:1.5rem!important}.me-xxl-5{margin-right:3rem!important}.me-xxl-auto{margin-right:auto!important}.mb-xxl-0{margin-bottom:0!important}.mb-xxl-1{margin-bottom:.25rem!important}.mb-xxl-2{margin-bottom:.5rem!important}.mb-xxl-3{margin-bottom:1rem!important}.mb-xxl-4{margin-bottom:1.5rem!important}.mb-xxl-5{margin-bottom:3rem!important}.mb-xxl-auto{margin-bottom:auto!important}.ms-xxl-0{margin-left:0!important}.ms-xxl-1{margin-left:.25rem!important}.ms-xxl-2{margin-left:.5rem!important}.ms-xxl-3{margin-left:1rem!important}.ms-xxl-4{margin-left:1.5rem!important}.ms-xxl-5{margin-left:3rem!important}.ms-xxl-auto{margin-left:auto!important}.p-xxl-0{padding:0!important}.p-xxl-1{padding:.25rem!important}.p-xxl-2{padding:.5rem!important}.p-xxl-3{padding:1rem!important}.p-xxl-4{padding:1.5rem!important}.p-xxl-5{padding:3rem!important}.px-xxl-0{padding-right:0!important;padding-left:0!important}.px-xxl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xxl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xxl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xxl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xxl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xxl-0{padding-top:0!important;padding-bottom:0!important}.py-xxl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xxl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xxl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xxl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xxl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xxl-0{padding-top:0!important}.pt-xxl-1{padding-top:.25rem!important}.pt-xxl-2{padding-top:.5rem!important}.pt-xxl-3{padding-top:1rem!important}.pt-xxl-4{padding-top:1.5rem!important}.pt-xxl-5{padding-top:3rem!important}.pe-xxl-0{padding-right:0!important}.pe-xxl-1{padding-right:.25rem!important}.pe-xxl-2{padding-right:.5rem!important}.pe-xxl-3{padding-right:1rem!important}.pe-xxl-4{padding-right:1.5rem!important}.pe-xxl-5{padding-right:3rem!important}.pb-xxl-0{padding-bottom:0!important}.pb-xxl-1{padding-bottom:.25rem!important}.pb-xxl-2{padding-bottom:.5rem!important}.pb-xxl-3{padding-bottom:1rem!important}.pb-xxl-4{padding-bottom:1.5rem!important}.pb-xxl-5{padding-bottom:3rem!important}.ps-xxl-0{padding-left:0!important}.ps-xxl-1{padding-left:.25rem!important}.ps-xxl-2{padding-left:.5rem!important}.ps-xxl-3{padding-left:1rem!important}.ps-xxl-4{padding-left:1.5rem!important}.ps-xxl-5{padding-left:3rem!important}.text-xxl-start{text-align:left!important}.text-xxl-end{text-align:right!important}.text-xxl-center{text-align:center!important}}@media (min-width:1200px){.fs-1{font-size:2.5rem!important}.fs-2{font-size:2rem!important}.fs-3{font-size:1.75rem!important}.fs-4{font-size:1.5rem!important}}@media print{.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-grid{display:grid!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:flex!important}.d-print-inline-flex{display:inline-flex!important}.d-print-none{display:none!important}} +/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css.map b/playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css.map new file mode 100644 index 0000000000..afcd9e33e9 --- /dev/null +++ b/playground/DashboardToy/DashboardToy.Frontend/wwwroot/css/bootstrap/bootstrap.min.css.map @@ -0,0 +1 @@ +{"version":3,"sources":["../../scss/bootstrap.scss","../../scss/_root.scss","../../scss/_reboot.scss","dist/css/bootstrap.css","../../scss/vendor/_rfs.scss","../../scss/mixins/_border-radius.scss","../../scss/_type.scss","../../scss/mixins/_lists.scss","../../scss/_images.scss","../../scss/mixins/_image.scss","../../scss/_containers.scss","../../scss/mixins/_container.scss","../../scss/mixins/_breakpoints.scss","../../scss/_grid.scss","../../scss/mixins/_grid.scss","../../scss/_tables.scss","../../scss/mixins/_table-variants.scss","../../scss/forms/_labels.scss","../../scss/forms/_form-text.scss","../../scss/forms/_form-control.scss","../../scss/mixins/_transition.scss","../../scss/mixins/_gradients.scss","../../scss/forms/_form-select.scss","../../scss/forms/_form-check.scss","../../scss/forms/_form-range.scss","../../scss/forms/_floating-labels.scss","../../scss/forms/_input-group.scss","../../scss/mixins/_forms.scss","../../scss/_buttons.scss","../../scss/mixins/_buttons.scss","../../scss/_transitions.scss","../../scss/_dropdown.scss","../../scss/mixins/_caret.scss","../../scss/_button-group.scss","../../scss/_nav.scss","../../scss/_navbar.scss","../../scss/_card.scss","../../scss/_accordion.scss","../../scss/_breadcrumb.scss","../../scss/_pagination.scss","../../scss/mixins/_pagination.scss","../../scss/_badge.scss","../../scss/_alert.scss","../../scss/mixins/_alert.scss","../../scss/_progress.scss","../../scss/_list-group.scss","../../scss/mixins/_list-group.scss","../../scss/_close.scss","../../scss/_toasts.scss","../../scss/_modal.scss","../../scss/mixins/_backdrop.scss","../../scss/_tooltip.scss","../../scss/mixins/_reset-text.scss","../../scss/_popover.scss","../../scss/_carousel.scss","../../scss/mixins/_clearfix.scss","../../scss/_spinners.scss","../../scss/_offcanvas.scss","../../scss/_placeholders.scss","../../scss/helpers/_colored-links.scss","../../scss/helpers/_ratio.scss","../../scss/helpers/_position.scss","../../scss/helpers/_stacks.scss","../../scss/helpers/_visually-hidden.scss","../../scss/mixins/_visually-hidden.scss","../../scss/helpers/_stretched-link.scss","../../scss/helpers/_text-truncation.scss","../../scss/mixins/_text-truncate.scss","../../scss/helpers/_vr.scss","../../scss/mixins/_utilities.scss","../../scss/utilities/_api.scss"],"names":[],"mappings":"iBAAA;;;;;ACAA,MAQI,UAAA,QAAA,YAAA,QAAA,YAAA,QAAA,UAAA,QAAA,SAAA,QAAA,YAAA,QAAA,YAAA,QAAA,WAAA,QAAA,UAAA,QAAA,UAAA,QAAA,WAAA,KAAA,UAAA,QAAA,eAAA,QAIA,cAAA,QAAA,cAAA,QAAA,cAAA,QAAA,cAAA,QAAA,cAAA,QAAA,cAAA,QAAA,cAAA,QAAA,cAAA,QAAA,cAAA,QAIA,aAAA,QAAA,eAAA,QAAA,aAAA,QAAA,UAAA,QAAA,aAAA,QAAA,YAAA,QAAA,WAAA,QAAA,UAAA,QAIA,iBAAA,EAAA,CAAA,GAAA,CAAA,IAAA,mBAAA,GAAA,CAAA,GAAA,CAAA,IAAA,iBAAA,EAAA,CAAA,GAAA,CAAA,GAAA,cAAA,EAAA,CAAA,GAAA,CAAA,IAAA,iBAAA,GAAA,CAAA,GAAA,CAAA,EAAA,gBAAA,GAAA,CAAA,EAAA,CAAA,GAAA,eAAA,GAAA,CAAA,GAAA,CAAA,IAAA,cAAA,EAAA,CAAA,EAAA,CAAA,GAGF,eAAA,GAAA,CAAA,GAAA,CAAA,IACA,eAAA,CAAA,CAAA,CAAA,CAAA,EACA,cAAA,EAAA,CAAA,EAAA,CAAA,GAMA,qBAAA,SAAA,CAAA,aAAA,CAAA,UAAA,CAAA,MAAA,CAAA,gBAAA,CAAA,KAAA,CAAA,WAAA,CAAA,iBAAA,CAAA,UAAA,CAAA,mBAAA,CAAA,gBAAA,CAAA,iBAAA,CAAA,mBACA,oBAAA,cAAA,CAAA,KAAA,CAAA,MAAA,CAAA,QAAA,CAAA,iBAAA,CAAA,aAAA,CAAA,UACA,cAAA,2EAQA,sBAAA,0BACA,oBAAA,KACA,sBAAA,IACA,sBAAA,IACA,gBAAA,QAIA,aAAA,KClCF,EC+CA,QADA,SD3CE,WAAA,WAeE,8CANJ,MAOM,gBAAA,QAcN,KACE,OAAA,EACA,YAAA,2BEmPI,UAAA,yBFjPJ,YAAA,2BACA,YAAA,2BACA,MAAA,qBACA,WAAA,0BACA,iBAAA,kBACA,yBAAA,KACA,4BAAA,YAUF,GACE,OAAA,KAAA,EACA,MAAA,QACA,iBAAA,aACA,OAAA,EACA,QAAA,IAGF,eACE,OAAA,IAUF,IAAA,IAAA,IAAA,IAAA,IAAA,IAAA,GAAA,GAAA,GAAA,GAAA,GAAA,GACE,WAAA,EACA,cAAA,MAGA,YAAA,IACA,YAAA,IAIF,IAAA,GEwMQ,UAAA,uBAlKJ,0BFtCJ,IAAA,GE+MQ,UAAA,QF1MR,IAAA,GEmMQ,UAAA,sBAlKJ,0BFjCJ,IAAA,GE0MQ,UAAA,MFrMR,IAAA,GE8LQ,UAAA,oBAlKJ,0BF5BJ,IAAA,GEqMQ,UAAA,SFhMR,IAAA,GEyLQ,UAAA,sBAlKJ,0BFvBJ,IAAA,GEgMQ,UAAA,QF3LR,IAAA,GEgLM,UAAA,QF3KN,IAAA,GE2KM,UAAA,KFhKN,EACE,WAAA,EACA,cAAA,KCmBF,6BDRA,YAEE,wBAAA,UAAA,OAAA,gBAAA,UAAA,OACA,OAAA,KACA,iCAAA,KAAA,yBAAA,KAMF,QACE,cAAA,KACA,WAAA,OACA,YAAA,QAMF,GCIA,GDFE,aAAA,KCQF,GDLA,GCIA,GDDE,WAAA,EACA,cAAA,KAGF,MCKA,MACA,MAFA,MDAE,cAAA,EAGF,GACE,YAAA,IAKF,GACE,cAAA,MACA,YAAA,EAMF,WACE,OAAA,EAAA,EAAA,KAQF,ECNA,ODQE,YAAA,OAQF,OAAA,ME4EM,UAAA,OFrEN,MAAA,KACE,QAAA,KACA,iBAAA,QASF,ICpBA,IDsBE,SAAA,SEwDI,UAAA,MFtDJ,YAAA,EACA,eAAA,SAGF,IAAM,OAAA,OACN,IAAM,IAAA,MAKN,EACE,MAAA,QACA,gBAAA,UAEA,QACE,MAAA,QAWF,2BAAA,iCAEE,MAAA,QACA,gBAAA,KCxBJ,KACA,ID8BA,IC7BA,KDiCE,YAAA,yBEcI,UAAA,IFZJ,UAAA,IACA,aAAA,cAOF,IACE,QAAA,MACA,WAAA,EACA,cAAA,KACA,SAAA,KEAI,UAAA,OFKJ,SELI,UAAA,QFOF,MAAA,QACA,WAAA,OAIJ,KEZM,UAAA,OFcJ,MAAA,QACA,UAAA,WAGA,OACE,MAAA,QAIJ,IACE,QAAA,MAAA,MExBI,UAAA,OF0BJ,MAAA,KACA,iBAAA,QG7SE,cAAA,MHgTF,QACE,QAAA,EE/BE,UAAA,IFiCF,YAAA,IASJ,OACE,OAAA,EAAA,EAAA,KAMF,ICjDA,IDmDE,eAAA,OAQF,MACE,aAAA,OACA,gBAAA,SAGF,QACE,YAAA,MACA,eAAA,MACA,MAAA,QACA,WAAA,KAOF,GAEE,WAAA,QACA,WAAA,qBCxDF,MAGA,GAFA,MAGA,GDuDA,MCzDA,GD+DE,aAAA,QACA,aAAA,MACA,aAAA,EAQF,MACE,QAAA,aAMF,OAEE,cAAA,EAQF,iCACE,QAAA,ECtEF,OD2EA,MCzEA,SADA,OAEA,SD6EE,OAAA,EACA,YAAA,QE9HI,UAAA,QFgIJ,YAAA,QAIF,OC5EA,OD8EE,eAAA,KAKF,cACE,OAAA,QAGF,OAGE,UAAA,OAGA,gBACE,QAAA,EAOJ,0CACE,QAAA,KClFF,cACA,aACA,cDwFA,OAIE,mBAAA,OCxFF,6BACA,4BACA,6BDyFI,sBACE,OAAA,QAON,mBACE,QAAA,EACA,aAAA,KAKF,SACE,OAAA,SAUF,SACE,UAAA,EACA,QAAA,EACA,OAAA,EACA,OAAA,EAQF,OACE,MAAA,KACA,MAAA,KACA,QAAA,EACA,cAAA,MEnNM,UAAA,sBFsNN,YAAA,QExXE,0BFiXJ,OExMQ,UAAA,QFiNN,SACE,MAAA,KChGJ,kCDuGA,uCCxGA,mCADA,+BAGA,oCAJA,6BAKA,mCD4GE,QAAA,EAGF,4BACE,OAAA,KASF,cACE,eAAA,KACA,mBAAA,UAmBF,4BACE,mBAAA,KAKF,+BACE,QAAA,EAMF,uBACE,KAAA,QAMF,6BACE,KAAA,QACA,mBAAA,OAKF,OACE,QAAA,aAKF,OACE,OAAA,EAOF,QACE,QAAA,UACA,OAAA,QAQF,SACE,eAAA,SAQF,SACE,QAAA,eInlBF,MFyQM,UAAA,QEvQJ,YAAA,IAKA,WFsQM,UAAA,uBEpQJ,YAAA,IACA,YAAA,IFiGA,0BEpGF,WF6QM,UAAA,ME7QN,WFsQM,UAAA,uBEpQJ,YAAA,IACA,YAAA,IFiGA,0BEpGF,WF6QM,UAAA,QE7QN,WFsQM,UAAA,uBEpQJ,YAAA,IACA,YAAA,IFiGA,0BEpGF,WF6QM,UAAA,ME7QN,WFsQM,UAAA,uBEpQJ,YAAA,IACA,YAAA,IFiGA,0BEpGF,WF6QM,UAAA,QE7QN,WFsQM,UAAA,uBEpQJ,YAAA,IACA,YAAA,IFiGA,0BEpGF,WF6QM,UAAA,ME7QN,WFsQM,UAAA,uBEpQJ,YAAA,IACA,YAAA,IFiGA,0BEpGF,WF6QM,UAAA,QEvPR,eCrDE,aAAA,EACA,WAAA,KDyDF,aC1DE,aAAA,EACA,WAAA,KD4DF,kBACE,QAAA,aAEA,mCACE,aAAA,MAUJ,YFsNM,UAAA,OEpNJ,eAAA,UAIF,YACE,cAAA,KF+MI,UAAA,QE5MJ,wBACE,cAAA,EAIJ,mBACE,WAAA,MACA,cAAA,KFqMI,UAAA,OEnMJ,MAAA,QAEA,2BACE,QAAA,KE9FJ,WCIE,UAAA,KAGA,OAAA,KDDF,eACE,QAAA,OACA,iBAAA,KACA,OAAA,IAAA,MAAA,QHGE,cAAA,OIRF,UAAA,KAGA,OAAA,KDcF,QAEE,QAAA,aAGF,YACE,cAAA,MACA,YAAA,EAGF,gBJ+PM,UAAA,OI7PJ,MAAA,QElCA,WPqmBF,iBAGA,cACA,cACA,cAHA,cADA,eQzmBE,MAAA,KACA,cAAA,0BACA,aAAA,0BACA,aAAA,KACA,YAAA,KCwDE,yBF5CE,WAAA,cACE,UAAA,OE2CJ,yBF5CE,WAAA,cAAA,cACE,UAAA,OE2CJ,yBF5CE,WAAA,cAAA,cAAA,cACE,UAAA,OE2CJ,0BF5CE,WAAA,cAAA,cAAA,cAAA,cACE,UAAA,QE2CJ,0BF5CE,WAAA,cAAA,cAAA,cAAA,cAAA,eACE,UAAA,QGfN,KCAA,cAAA,OACA,cAAA,EACA,QAAA,KACA,UAAA,KACA,WAAA,8BACA,aAAA,+BACA,YAAA,+BDHE,OCYF,YAAA,EACA,MAAA,KACA,UAAA,KACA,cAAA,8BACA,aAAA,8BACA,WAAA,mBA+CI,KACE,KAAA,EAAA,EAAA,GAGF,iBApCJ,KAAA,EAAA,EAAA,KACA,MAAA,KAcA,cACE,KAAA,EAAA,EAAA,KACA,MAAA,KAFF,cACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,cACE,KAAA,EAAA,EAAA,KACA,MAAA,eAFF,cACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,cACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,cACE,KAAA,EAAA,EAAA,KACA,MAAA,eA+BE,UAhDJ,KAAA,EAAA,EAAA,KACA,MAAA,KAqDQ,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,YA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,OAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,QAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,QAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,QAhEN,KAAA,EAAA,EAAA,KACA,MAAA,KAuEQ,UAxDV,YAAA,YAwDU,UAxDV,YAAA,aAwDU,UAxDV,YAAA,IAwDU,UAxDV,YAAA,aAwDU,UAxDV,YAAA,aAwDU,UAxDV,YAAA,IAwDU,UAxDV,YAAA,aAwDU,UAxDV,YAAA,aAwDU,UAxDV,YAAA,IAwDU,WAxDV,YAAA,aAwDU,WAxDV,YAAA,aAmEM,KXusBR,MWrsBU,cAAA,EAGF,KXusBR,MWrsBU,cAAA,EAPF,KXitBR,MW/sBU,cAAA,QAGF,KXitBR,MW/sBU,cAAA,QAPF,KX2tBR,MWztBU,cAAA,OAGF,KX2tBR,MWztBU,cAAA,OAPF,KXquBR,MWnuBU,cAAA,KAGF,KXquBR,MWnuBU,cAAA,KAPF,KX+uBR,MW7uBU,cAAA,OAGF,KX+uBR,MW7uBU,cAAA,OAPF,KXyvBR,MWvvBU,cAAA,KAGF,KXyvBR,MWvvBU,cAAA,KFzDN,yBESE,QACE,KAAA,EAAA,EAAA,GAGF,oBApCJ,KAAA,EAAA,EAAA,KACA,MAAA,KAcA,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,KAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eA+BE,aAhDJ,KAAA,EAAA,EAAA,KACA,MAAA,KAqDQ,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,YA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,KAuEQ,aAxDV,YAAA,EAwDU,aAxDV,YAAA,YAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,aAmEM,QX45BR,SW15BU,cAAA,EAGF,QX45BR,SW15BU,cAAA,EAPF,QXs6BR,SWp6BU,cAAA,QAGF,QXs6BR,SWp6BU,cAAA,QAPF,QXg7BR,SW96BU,cAAA,OAGF,QXg7BR,SW96BU,cAAA,OAPF,QX07BR,SWx7BU,cAAA,KAGF,QX07BR,SWx7BU,cAAA,KAPF,QXo8BR,SWl8BU,cAAA,OAGF,QXo8BR,SWl8BU,cAAA,OAPF,QX88BR,SW58BU,cAAA,KAGF,QX88BR,SW58BU,cAAA,MFzDN,yBESE,QACE,KAAA,EAAA,EAAA,GAGF,oBApCJ,KAAA,EAAA,EAAA,KACA,MAAA,KAcA,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,KAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eA+BE,aAhDJ,KAAA,EAAA,EAAA,KACA,MAAA,KAqDQ,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,YA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,KAuEQ,aAxDV,YAAA,EAwDU,aAxDV,YAAA,YAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,aAmEM,QXinCR,SW/mCU,cAAA,EAGF,QXinCR,SW/mCU,cAAA,EAPF,QX2nCR,SWznCU,cAAA,QAGF,QX2nCR,SWznCU,cAAA,QAPF,QXqoCR,SWnoCU,cAAA,OAGF,QXqoCR,SWnoCU,cAAA,OAPF,QX+oCR,SW7oCU,cAAA,KAGF,QX+oCR,SW7oCU,cAAA,KAPF,QXypCR,SWvpCU,cAAA,OAGF,QXypCR,SWvpCU,cAAA,OAPF,QXmqCR,SWjqCU,cAAA,KAGF,QXmqCR,SWjqCU,cAAA,MFzDN,yBESE,QACE,KAAA,EAAA,EAAA,GAGF,oBApCJ,KAAA,EAAA,EAAA,KACA,MAAA,KAcA,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,KAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eA+BE,aAhDJ,KAAA,EAAA,EAAA,KACA,MAAA,KAqDQ,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,YA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,KAuEQ,aAxDV,YAAA,EAwDU,aAxDV,YAAA,YAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,aAmEM,QXs0CR,SWp0CU,cAAA,EAGF,QXs0CR,SWp0CU,cAAA,EAPF,QXg1CR,SW90CU,cAAA,QAGF,QXg1CR,SW90CU,cAAA,QAPF,QX01CR,SWx1CU,cAAA,OAGF,QX01CR,SWx1CU,cAAA,OAPF,QXo2CR,SWl2CU,cAAA,KAGF,QXo2CR,SWl2CU,cAAA,KAPF,QX82CR,SW52CU,cAAA,OAGF,QX82CR,SW52CU,cAAA,OAPF,QXw3CR,SWt3CU,cAAA,KAGF,QXw3CR,SWt3CU,cAAA,MFzDN,0BESE,QACE,KAAA,EAAA,EAAA,GAGF,oBApCJ,KAAA,EAAA,EAAA,KACA,MAAA,KAcA,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,KAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,iBACE,KAAA,EAAA,EAAA,KACA,MAAA,eA+BE,aAhDJ,KAAA,EAAA,EAAA,KACA,MAAA,KAqDQ,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,YA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,UAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,KAuEQ,aAxDV,YAAA,EAwDU,aAxDV,YAAA,YAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,aAwDU,aAxDV,YAAA,IAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,aAmEM,QX2hDR,SWzhDU,cAAA,EAGF,QX2hDR,SWzhDU,cAAA,EAPF,QXqiDR,SWniDU,cAAA,QAGF,QXqiDR,SWniDU,cAAA,QAPF,QX+iDR,SW7iDU,cAAA,OAGF,QX+iDR,SW7iDU,cAAA,OAPF,QXyjDR,SWvjDU,cAAA,KAGF,QXyjDR,SWvjDU,cAAA,KAPF,QXmkDR,SWjkDU,cAAA,OAGF,QXmkDR,SWjkDU,cAAA,OAPF,QX6kDR,SW3kDU,cAAA,KAGF,QX6kDR,SW3kDU,cAAA,MFzDN,0BESE,SACE,KAAA,EAAA,EAAA,GAGF,qBApCJ,KAAA,EAAA,EAAA,KACA,MAAA,KAcA,kBACE,KAAA,EAAA,EAAA,KACA,MAAA,KAFF,kBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,kBACE,KAAA,EAAA,EAAA,KACA,MAAA,eAFF,kBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,kBACE,KAAA,EAAA,EAAA,KACA,MAAA,IAFF,kBACE,KAAA,EAAA,EAAA,KACA,MAAA,eA+BE,cAhDJ,KAAA,EAAA,EAAA,KACA,MAAA,KAqDQ,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,YA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,WAhEN,KAAA,EAAA,EAAA,KACA,MAAA,IA+DM,YAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,YAhEN,KAAA,EAAA,EAAA,KACA,MAAA,aA+DM,YAhEN,KAAA,EAAA,EAAA,KACA,MAAA,KAuEQ,cAxDV,YAAA,EAwDU,cAxDV,YAAA,YAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,IAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,IAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,aAwDU,cAxDV,YAAA,IAwDU,eAxDV,YAAA,aAwDU,eAxDV,YAAA,aAmEM,SXgvDR,UW9uDU,cAAA,EAGF,SXgvDR,UW9uDU,cAAA,EAPF,SX0vDR,UWxvDU,cAAA,QAGF,SX0vDR,UWxvDU,cAAA,QAPF,SXowDR,UWlwDU,cAAA,OAGF,SXowDR,UWlwDU,cAAA,OAPF,SX8wDR,UW5wDU,cAAA,KAGF,SX8wDR,UW5wDU,cAAA,KAPF,SXwxDR,UWtxDU,cAAA,OAGF,SXwxDR,UWtxDU,cAAA,OAPF,SXkyDR,UWhyDU,cAAA,KAGF,SXkyDR,UWhyDU,cAAA,MCpHV,OACE,cAAA,YACA,qBAAA,YACA,yBAAA,QACA,sBAAA,oBACA,wBAAA,QACA,qBAAA,mBACA,uBAAA,QACA,oBAAA,qBAEA,MAAA,KACA,cAAA,KACA,MAAA,QACA,eAAA,IACA,aAAA,QAOA,yBACE,QAAA,MAAA,MACA,iBAAA,mBACA,oBAAA,IACA,WAAA,MAAA,EAAA,EAAA,EAAA,OAAA,0BAGF,aACE,eAAA,QAGF,aACE,eAAA,OAIF,uCACE,oBAAA,aASJ,aACE,aAAA,IAUA,4BACE,QAAA,OAAA,OAeF,gCACE,aAAA,IAAA,EAGA,kCACE,aAAA,EAAA,IAOJ,oCACE,oBAAA,EASF,yCACE,qBAAA,2BACA,MAAA,8BAQJ,cACE,qBAAA,0BACA,MAAA,6BAQA,4BACE,qBAAA,yBACA,MAAA,4BCxHF,eAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QAfF,iBAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QAfF,eAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QAfF,YAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QAfF,eAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QAfF,cAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QAfF,aAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QAfF,YAME,cAAA,QACA,sBAAA,QACA,yBAAA,KACA,qBAAA,QACA,wBAAA,KACA,oBAAA,QACA,uBAAA,KAEA,MAAA,KACA,aAAA,QDgIA,kBACE,WAAA,KACA,2BAAA,MHvEF,4BGqEA,qBACE,WAAA,KACA,2BAAA,OHvEF,4BGqEA,qBACE,WAAA,KACA,2BAAA,OHvEF,4BGqEA,qBACE,WAAA,KACA,2BAAA,OHvEF,6BGqEA,qBACE,WAAA,KACA,2BAAA,OHvEF,6BGqEA,sBACE,WAAA,KACA,2BAAA,OE/IN,YACE,cAAA,MASF,gBACE,YAAA,oBACA,eAAA,oBACA,cAAA,EboRI,UAAA,QahRJ,YAAA,IAIF,mBACE,YAAA,kBACA,eAAA,kBb0QI,UAAA,QatQN,mBACE,YAAA,mBACA,eAAA,mBboQI,UAAA,QcjSN,WACE,WAAA,OdgSI,UAAA,Oc5RJ,MAAA,QCLF,cACE,QAAA,MACA,MAAA,KACA,QAAA,QAAA,Of8RI,UAAA,Ke3RJ,YAAA,IACA,YAAA,IACA,MAAA,QACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,QACA,mBAAA,KAAA,gBAAA,KAAA,WAAA,KdGE,cAAA,OeHE,WAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAIA,uCDhBN,cCiBQ,WAAA,MDGN,yBACE,SAAA,OAEA,wDACE,OAAA,QAKJ,oBACE,MAAA,QACA,iBAAA,KACA,aAAA,QACA,QAAA,EAKE,WAAA,EAAA,EAAA,EAAA,OAAA,qBAOJ,2CAEE,OAAA,MAIF,gCACE,MAAA,QAEA,QAAA,EAHF,2BACE,MAAA,QAEA,QAAA,EAQF,uBAAA,wBAEE,iBAAA,QAGA,QAAA,EAIF,oCACE,QAAA,QAAA,OACA,OAAA,SAAA,QACA,mBAAA,OAAA,kBAAA,OACA,MAAA,QE3EF,iBAAA,QF6EE,eAAA,KACA,aAAA,QACA,aAAA,MACA,aAAA,EACA,wBAAA,IACA,cAAA,ECtEE,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAIA,uCDuDJ,oCCtDM,WAAA,MDqEN,yEACE,iBAAA,QAGF,0CACE,QAAA,QAAA,OACA,OAAA,SAAA,QACA,mBAAA,OAAA,kBAAA,OACA,MAAA,QE9FF,iBAAA,QFgGE,eAAA,KACA,aAAA,QACA,aAAA,MACA,aAAA,EACA,wBAAA,IACA,cAAA,ECzFE,mBAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAAA,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAIA,uCD0EJ,0CCzEM,mBAAA,KAAA,WAAA,MDwFN,+EACE,iBAAA,QASJ,wBACE,QAAA,MACA,MAAA,KACA,QAAA,QAAA,EACA,cAAA,EACA,YAAA,IACA,MAAA,QACA,iBAAA,YACA,OAAA,MAAA,YACA,aAAA,IAAA,EAEA,wCAAA,wCAEE,cAAA,EACA,aAAA,EAWJ,iBACE,WAAA,0BACA,QAAA,OAAA,MfmJI,UAAA,QClRF,cAAA,McmIF,uCACE,QAAA,OAAA,MACA,OAAA,QAAA,OACA,mBAAA,MAAA,kBAAA,MAGF,6CACE,QAAA,OAAA,MACA,OAAA,QAAA,OACA,mBAAA,MAAA,kBAAA,MAIJ,iBACE,WAAA,yBACA,QAAA,MAAA,KfgII,UAAA,QClRF,cAAA,McsJF,uCACE,QAAA,MAAA,KACA,OAAA,OAAA,MACA,mBAAA,KAAA,kBAAA,KAGF,6CACE,QAAA,MAAA,KACA,OAAA,OAAA,MACA,mBAAA,KAAA,kBAAA,KAQF,sBACE,WAAA,2BAGF,yBACE,WAAA,0BAGF,yBACE,WAAA,yBAKJ,oBACE,MAAA,KACA,OAAA,KACA,QAAA,QAEA,mDACE,OAAA,QAGF,uCACE,OAAA,Md/LA,cAAA,OcmMF,0CACE,OAAA,MdpMA,cAAA,OiBdJ,aACE,QAAA,MACA,MAAA,KACA,QAAA,QAAA,QAAA,QAAA,OAEA,mBAAA,oBlB2RI,UAAA,KkBxRJ,YAAA,IACA,YAAA,IACA,MAAA,QACA,iBAAA,KACA,iBAAA,gOACA,kBAAA,UACA,oBAAA,MAAA,OAAA,OACA,gBAAA,KAAA,KACA,OAAA,IAAA,MAAA,QjBFE,cAAA,OeHE,WAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YESJ,mBAAA,KAAA,gBAAA,KAAA,WAAA,KFLI,uCEfN,aFgBQ,WAAA,MEMN,mBACE,aAAA,QACA,QAAA,EAKE,WAAA,EAAA,EAAA,EAAA,OAAA,qBAIJ,uBAAA,mCAEE,cAAA,OACA,iBAAA,KAGF,sBAEE,iBAAA,QAKF,4BACE,MAAA,YACA,YAAA,EAAA,EAAA,EAAA,QAIJ,gBACE,YAAA,OACA,eAAA,OACA,aAAA,MlByOI,UAAA,QkBrON,gBACE,YAAA,MACA,eAAA,MACA,aAAA,KlBkOI,UAAA,QmBjSN,YACE,QAAA,MACA,WAAA,OACA,aAAA,MACA,cAAA,QAEA,8BACE,MAAA,KACA,YAAA,OAIJ,kBACE,MAAA,IACA,OAAA,IACA,WAAA,MACA,eAAA,IACA,iBAAA,KACA,kBAAA,UACA,oBAAA,OACA,gBAAA,QACA,OAAA,IAAA,MAAA,gBACA,mBAAA,KAAA,gBAAA,KAAA,WAAA,KACA,2BAAA,MAAA,aAAA,MAGA,iClBXE,cAAA,MkBeF,8BAEE,cAAA,IAGF,yBACE,OAAA,gBAGF,wBACE,aAAA,QACA,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,OAAA,qBAGF,0BACE,iBAAA,QACA,aAAA,QAEA,yCAII,iBAAA,8NAIJ,sCAII,iBAAA,sIAKN,+CACE,iBAAA,QACA,aAAA,QAKE,iBAAA,wNAIJ,2BACE,eAAA,KACA,OAAA,KACA,QAAA,GAOA,6CAAA,8CACE,QAAA,GAcN,aACE,aAAA,MAEA,+BACE,MAAA,IACA,YAAA,OACA,iBAAA,uJACA,oBAAA,KAAA,OlB9FA,cAAA,IeHE,WAAA,oBAAA,KAAA,YAIA,uCGyFJ,+BHxFM,WAAA,MGgGJ,qCACE,iBAAA,yIAGF,uCACE,oBAAA,MAAA,OAKE,iBAAA,sIAMR,mBACE,QAAA,aACA,aAAA,KAGF,WACE,SAAA,SACA,KAAA,cACA,eAAA,KAIE,yBAAA,0BACE,eAAA,KACA,OAAA,KACA,QAAA,IC9IN,YACE,MAAA,KACA,OAAA,OACA,QAAA,EACA,iBAAA,YACA,mBAAA,KAAA,gBAAA,KAAA,WAAA,KAEA,kBACE,QAAA,EAIA,wCAA0B,WAAA,EAAA,EAAA,EAAA,IAAA,IAAA,CAAA,EAAA,EAAA,EAAA,OAAA,qBAC1B,oCAA0B,WAAA,EAAA,EAAA,EAAA,IAAA,IAAA,CAAA,EAAA,EAAA,EAAA,OAAA,qBAG5B,8BACE,OAAA,EAGF,kCACE,MAAA,KACA,OAAA,KACA,WAAA,QHzBF,iBAAA,QG2BE,OAAA,EnBZA,cAAA,KeHE,mBAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAAA,WAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YImBF,mBAAA,KAAA,WAAA,KJfE,uCIMJ,kCJLM,mBAAA,KAAA,WAAA,MIgBJ,yCHjCF,iBAAA,QGsCA,2CACE,MAAA,KACA,OAAA,MACA,MAAA,YACA,OAAA,QACA,iBAAA,QACA,aAAA,YnB7BA,cAAA,KmBkCF,8BACE,MAAA,KACA,OAAA,KHnDF,iBAAA,QGqDE,OAAA,EnBtCA,cAAA,KeHE,gBAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAAA,WAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YI6CF,gBAAA,KAAA,WAAA,KJzCE,uCIiCJ,8BJhCM,gBAAA,KAAA,WAAA,MI0CJ,qCH3DF,iBAAA,QGgEA,8BACE,MAAA,KACA,OAAA,MACA,MAAA,YACA,OAAA,QACA,iBAAA,QACA,aAAA,YnBvDA,cAAA,KmB4DF,qBACE,eAAA,KAEA,2CACE,iBAAA,QAGF,uCACE,iBAAA,QCvFN,eACE,SAAA,SAEA,6BtB+iFF,4BsB7iFI,OAAA,mBACA,YAAA,KAGF,qBACE,SAAA,SACA,IAAA,EACA,KAAA,EACA,OAAA,KACA,QAAA,KAAA,OACA,eAAA,KACA,OAAA,IAAA,MAAA,YACA,iBAAA,EAAA,ELDE,WAAA,QAAA,IAAA,WAAA,CAAA,UAAA,IAAA,YAIA,uCKXJ,qBLYM,WAAA,MKCN,6BACE,QAAA,KAAA,OAEA,+CACE,MAAA,YADF,0CACE,MAAA,YAGF,0DAEE,YAAA,SACA,eAAA,QAHF,mCAAA,qDAEE,YAAA,SACA,eAAA,QAGF,8CACE,YAAA,SACA,eAAA,QAIJ,4BACE,YAAA,SACA,eAAA,QAMA,gEACE,QAAA,IACA,UAAA,WAAA,mBAAA,mBAFF,yCtBmjFJ,2DACA,kCsBnjFM,QAAA,IACA,UAAA,WAAA,mBAAA,mBAKF,oDACE,QAAA,IACA,UAAA,WAAA,mBAAA,mBCtDN,aACE,SAAA,SACA,QAAA,KACA,UAAA,KACA,YAAA,QACA,MAAA,KAEA,2BvB2mFF,0BuBzmFI,SAAA,SACA,KAAA,EAAA,EAAA,KACA,MAAA,GACA,UAAA,EAIF,iCvBymFF,gCuBvmFI,QAAA,EAMF,kBACE,SAAA,SACA,QAAA,EAEA,wBACE,QAAA,EAWN,kBACE,QAAA,KACA,YAAA,OACA,QAAA,QAAA,OtBsPI,UAAA,KsBpPJ,YAAA,IACA,YAAA,IACA,MAAA,QACA,WAAA,OACA,YAAA,OACA,iBAAA,QACA,OAAA,IAAA,MAAA,QrBpCE,cAAA,OFuoFJ,qBuBzlFA,8BvBulFA,6BACA,kCuBplFE,QAAA,MAAA,KtBgOI,UAAA,QClRF,cAAA,MFgpFJ,qBuBzlFA,8BvBulFA,6BACA,kCuBplFE,QAAA,OAAA,MtBuNI,UAAA,QClRF,cAAA,MqBgEJ,6BvBulFA,6BuBrlFE,cAAA,KvB0lFF,uEuB7kFI,8FrB/DA,wBAAA,EACA,2BAAA,EFgpFJ,iEuB3kFI,2FrBtEA,wBAAA,EACA,2BAAA,EqBgFF,0IACE,YAAA,KrBpEA,uBAAA,EACA,0BAAA,EsBzBF,gBACE,QAAA,KACA,MAAA,KACA,WAAA,OvByQE,UAAA,OuBtQF,MAAA,QAGF,eACE,SAAA,SACA,IAAA,KACA,QAAA,EACA,QAAA,KACA,UAAA,KACA,QAAA,OAAA,MACA,WAAA,MvB4PE,UAAA,QuBzPF,MAAA,KACA,iBAAA,mBtB1BA,cAAA,OFmsFJ,0BACA,yBwBrqFI,sCxBmqFJ,qCwBjqFM,QAAA,MA9CF,uBAAA,mCAoDE,aAAA,QAGE,cAAA,qBACA,iBAAA,2OACA,kBAAA,UACA,oBAAA,MAAA,wBAAA,OACA,gBAAA,sBAAA,sBAGF,6BAAA,yCACE,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,OAAA,oBAhEJ,2CAAA,+BAyEI,cAAA,qBACA,oBAAA,IAAA,wBAAA,MAAA,wBA1EJ,sBAAA,kCAiFE,aAAA,QAGE,kDAAA,gDAAA,8DAAA,4DAEE,cAAA,SACA,iBAAA,+NAAA,CAAA,2OACA,oBAAA,MAAA,OAAA,MAAA,CAAA,OAAA,MAAA,QACA,gBAAA,KAAA,IAAA,CAAA,sBAAA,sBAIJ,4BAAA,wCACE,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,OAAA,oBA/FJ,2BAAA,uCAsGE,aAAA,QAEA,mCAAA,+CACE,iBAAA,QAGF,iCAAA,6CACE,WAAA,EAAA,EAAA,EAAA,OAAA,oBAGF,6CAAA,yDACE,MAAA,QAKJ,qDACE,YAAA,KAvHF,oCxBwwFJ,mCwBxwFI,gDxBuwFJ,+CwBxoFQ,QAAA,EAIF,0CxB0oFN,yCwB1oFM,sDxByoFN,qDwBxoFQ,QAAA,EAjHN,kBACE,QAAA,KACA,MAAA,KACA,WAAA,OvByQE,UAAA,OuBtQF,MAAA,QAGF,iBACE,SAAA,SACA,IAAA,KACA,QAAA,EACA,QAAA,KACA,UAAA,KACA,QAAA,OAAA,MACA,WAAA,MvB4PE,UAAA,QuBzPF,MAAA,KACA,iBAAA,mBtB1BA,cAAA,OF4xFJ,8BACA,6BwB9vFI,0CxB4vFJ,yCwB1vFM,QAAA,MA9CF,yBAAA,qCAoDE,aAAA,QAGE,cAAA,qBACA,iBAAA,2TACA,kBAAA,UACA,oBAAA,MAAA,wBAAA,OACA,gBAAA,sBAAA,sBAGF,+BAAA,2CACE,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,OAAA,oBAhEJ,6CAAA,iCAyEI,cAAA,qBACA,oBAAA,IAAA,wBAAA,MAAA,wBA1EJ,wBAAA,oCAiFE,aAAA,QAGE,oDAAA,kDAAA,gEAAA,8DAEE,cAAA,SACA,iBAAA,+NAAA,CAAA,2TACA,oBAAA,MAAA,OAAA,MAAA,CAAA,OAAA,MAAA,QACA,gBAAA,KAAA,IAAA,CAAA,sBAAA,sBAIJ,8BAAA,0CACE,aAAA,QACA,WAAA,EAAA,EAAA,EAAA,OAAA,oBA/FJ,6BAAA,yCAsGE,aAAA,QAEA,qCAAA,iDACE,iBAAA,QAGF,mCAAA,+CACE,WAAA,EAAA,EAAA,EAAA,OAAA,oBAGF,+CAAA,2DACE,MAAA,QAKJ,uDACE,YAAA,KAvHF,sCxBi2FJ,qCwBj2FI,kDxBg2FJ,iDwB/tFQ,QAAA,EAEF,4CxBmuFN,2CwBnuFM,wDxBkuFN,uDwBjuFQ,QAAA,ECtIR,KACE,QAAA,aAEA,YAAA,IACA,YAAA,IACA,MAAA,QACA,WAAA,OACA,gBAAA,KAEA,eAAA,OACA,OAAA,QACA,oBAAA,KAAA,iBAAA,KAAA,YAAA,KACA,iBAAA,YACA,OAAA,IAAA,MAAA,YC8GA,QAAA,QAAA,OzBsKI,UAAA,KClRF,cAAA,OeHE,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAIA,uCQhBN,KRiBQ,WAAA,MQAN,WACE,MAAA,QAIF,sBAAA,WAEE,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,OAAA,qBAcF,cAAA,cAAA,uBAGE,eAAA,KACA,QAAA,IAYF,aCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,mBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,8BAAA,mBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,oBAIJ,+BAAA,gCAAA,oBAAA,oBAAA,mCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,qCAAA,sCAAA,0BAAA,0BAAA,yCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,oBAKN,sBAAA,sBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDZF,eCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,qBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,gCAAA,qBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,qBAIJ,iCAAA,kCAAA,sBAAA,sBAAA,qCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,uCAAA,wCAAA,4BAAA,4BAAA,2CAKI,WAAA,EAAA,EAAA,EAAA,OAAA,qBAKN,wBAAA,wBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDZF,aCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,mBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,8BAAA,mBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,oBAIJ,+BAAA,gCAAA,oBAAA,oBAAA,mCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,qCAAA,sCAAA,0BAAA,0BAAA,yCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,oBAKN,sBAAA,sBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDZF,UCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,gBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,2BAAA,gBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,oBAIJ,4BAAA,6BAAA,iBAAA,iBAAA,gCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,kCAAA,mCAAA,uBAAA,uBAAA,sCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,oBAKN,mBAAA,mBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDZF,aCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,mBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,8BAAA,mBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,mBAIJ,+BAAA,gCAAA,oBAAA,oBAAA,mCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,qCAAA,sCAAA,0BAAA,0BAAA,yCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,mBAKN,sBAAA,sBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDZF,YCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,kBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,6BAAA,kBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,mBAIJ,8BAAA,+BAAA,mBAAA,mBAAA,kCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,oCAAA,qCAAA,yBAAA,yBAAA,wCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,mBAKN,qBAAA,qBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDZF,WCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,iBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,4BAAA,iBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,qBAIJ,6BAAA,8BAAA,kBAAA,kBAAA,iCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,mCAAA,oCAAA,wBAAA,wBAAA,uCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,qBAKN,oBAAA,oBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDZF,UCvCA,MAAA,KRhBA,iBAAA,QQkBA,aAAA,QAGA,gBACE,MAAA,KRtBF,iBAAA,QQwBE,aAAA,QAGF,2BAAA,gBAEE,MAAA,KR7BF,iBAAA,QQ+BE,aAAA,QAKE,WAAA,EAAA,EAAA,EAAA,OAAA,kBAIJ,4BAAA,6BAAA,iBAAA,iBAAA,gCAKE,MAAA,KACA,iBAAA,QAGA,aAAA,QAEA,kCAAA,mCAAA,uBAAA,uBAAA,sCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,kBAKN,mBAAA,mBAEE,MAAA,KACA,iBAAA,QAGA,aAAA,QDNF,qBCmBA,MAAA,QACA,aAAA,QAEA,2BACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,sCAAA,2BAEE,WAAA,EAAA,EAAA,EAAA,OAAA,oBAGF,uCAAA,wCAAA,4BAAA,0CAAA,4BAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,6CAAA,8CAAA,kCAAA,gDAAA,kCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,oBAKN,8BAAA,8BAEE,MAAA,QACA,iBAAA,YDvDF,uBCmBA,MAAA,QACA,aAAA,QAEA,6BACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,wCAAA,6BAEE,WAAA,EAAA,EAAA,EAAA,OAAA,qBAGF,yCAAA,0CAAA,8BAAA,4CAAA,8BAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,+CAAA,gDAAA,oCAAA,kDAAA,oCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,qBAKN,gCAAA,gCAEE,MAAA,QACA,iBAAA,YDvDF,qBCmBA,MAAA,QACA,aAAA,QAEA,2BACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,sCAAA,2BAEE,WAAA,EAAA,EAAA,EAAA,OAAA,mBAGF,uCAAA,wCAAA,4BAAA,0CAAA,4BAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,6CAAA,8CAAA,kCAAA,gDAAA,kCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,mBAKN,8BAAA,8BAEE,MAAA,QACA,iBAAA,YDvDF,kBCmBA,MAAA,QACA,aAAA,QAEA,wBACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,mCAAA,wBAEE,WAAA,EAAA,EAAA,EAAA,OAAA,oBAGF,oCAAA,qCAAA,yBAAA,uCAAA,yBAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,0CAAA,2CAAA,+BAAA,6CAAA,+BAKI,WAAA,EAAA,EAAA,EAAA,OAAA,oBAKN,2BAAA,2BAEE,MAAA,QACA,iBAAA,YDvDF,qBCmBA,MAAA,QACA,aAAA,QAEA,2BACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,sCAAA,2BAEE,WAAA,EAAA,EAAA,EAAA,OAAA,mBAGF,uCAAA,wCAAA,4BAAA,0CAAA,4BAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,6CAAA,8CAAA,kCAAA,gDAAA,kCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,mBAKN,8BAAA,8BAEE,MAAA,QACA,iBAAA,YDvDF,oBCmBA,MAAA,QACA,aAAA,QAEA,0BACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,qCAAA,0BAEE,WAAA,EAAA,EAAA,EAAA,OAAA,mBAGF,sCAAA,uCAAA,2BAAA,yCAAA,2BAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,4CAAA,6CAAA,iCAAA,+CAAA,iCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,mBAKN,6BAAA,6BAEE,MAAA,QACA,iBAAA,YDvDF,mBCmBA,MAAA,QACA,aAAA,QAEA,yBACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,oCAAA,yBAEE,WAAA,EAAA,EAAA,EAAA,OAAA,qBAGF,qCAAA,sCAAA,0BAAA,wCAAA,0BAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,2CAAA,4CAAA,gCAAA,8CAAA,gCAKI,WAAA,EAAA,EAAA,EAAA,OAAA,qBAKN,4BAAA,4BAEE,MAAA,QACA,iBAAA,YDvDF,kBCmBA,MAAA,QACA,aAAA,QAEA,wBACE,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,mCAAA,wBAEE,WAAA,EAAA,EAAA,EAAA,OAAA,kBAGF,oCAAA,qCAAA,yBAAA,uCAAA,yBAKE,MAAA,KACA,iBAAA,QACA,aAAA,QAEA,0CAAA,2CAAA,+BAAA,6CAAA,+BAKI,WAAA,EAAA,EAAA,EAAA,OAAA,kBAKN,2BAAA,2BAEE,MAAA,QACA,iBAAA,YD3CJ,UACE,YAAA,IACA,MAAA,QACA,gBAAA,UAEA,gBACE,MAAA,QAQF,mBAAA,mBAEE,MAAA,QAWJ,mBAAA,QCuBE,QAAA,MAAA,KzBsKI,UAAA,QClRF,cAAA,MuByFJ,mBAAA,QCmBE,QAAA,OAAA,MzBsKI,UAAA,QClRF,cAAA,MyBnBJ,MVgBM,WAAA,QAAA,KAAA,OAIA,uCUpBN,MVqBQ,WAAA,MUlBN,iBACE,QAAA,EAMF,qBACE,QAAA,KAIJ,YACE,OAAA,EACA,SAAA,OVDI,WAAA,OAAA,KAAA,KAIA,uCULN,YVMQ,WAAA,MUDN,gCACE,MAAA,EACA,OAAA,KVNE,WAAA,MAAA,KAAA,KAIA,uCUAJ,gCVCM,WAAA,MjBs3GR,UADA,SAEA,W4B34GA,QAIE,SAAA,SAGF,iBACE,YAAA,OCqBE,wBACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAhCJ,WAAA,KAAA,MACA,aAAA,KAAA,MAAA,YACA,cAAA,EACA,YAAA,KAAA,MAAA,YAqDE,8BACE,YAAA,ED3CN,eACE,SAAA,SACA,QAAA,KACA,QAAA,KACA,UAAA,MACA,QAAA,MAAA,EACA,OAAA,E3B+QI,UAAA,K2B7QJ,MAAA,QACA,WAAA,KACA,WAAA,KACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,gB1BVE,cAAA,O0BcF,+BACE,IAAA,KACA,KAAA,EACA,WAAA,QAYA,qBACE,cAAA,MAEA,qCACE,MAAA,KACA,KAAA,EAIJ,mBACE,cAAA,IAEA,mCACE,MAAA,EACA,KAAA,KnBCJ,yBmBfA,wBACE,cAAA,MAEA,wCACE,MAAA,KACA,KAAA,EAIJ,sBACE,cAAA,IAEA,sCACE,MAAA,EACA,KAAA,MnBCJ,yBmBfA,wBACE,cAAA,MAEA,wCACE,MAAA,KACA,KAAA,EAIJ,sBACE,cAAA,IAEA,sCACE,MAAA,EACA,KAAA,MnBCJ,yBmBfA,wBACE,cAAA,MAEA,wCACE,MAAA,KACA,KAAA,EAIJ,sBACE,cAAA,IAEA,sCACE,MAAA,EACA,KAAA,MnBCJ,0BmBfA,wBACE,cAAA,MAEA,wCACE,MAAA,KACA,KAAA,EAIJ,sBACE,cAAA,IAEA,sCACE,MAAA,EACA,KAAA,MnBCJ,0BmBfA,yBACE,cAAA,MAEA,yCACE,MAAA,KACA,KAAA,EAIJ,uBACE,cAAA,IAEA,uCACE,MAAA,EACA,KAAA,MAUN,uCACE,IAAA,KACA,OAAA,KACA,WAAA,EACA,cAAA,QC9CA,gCACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAzBJ,WAAA,EACA,aAAA,KAAA,MAAA,YACA,cAAA,KAAA,MACA,YAAA,KAAA,MAAA,YA8CE,sCACE,YAAA,ED0BJ,wCACE,IAAA,EACA,MAAA,KACA,KAAA,KACA,WAAA,EACA,YAAA,QC5DA,iCACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAlBJ,WAAA,KAAA,MAAA,YACA,aAAA,EACA,cAAA,KAAA,MAAA,YACA,YAAA,KAAA,MAuCE,uCACE,YAAA,EDoCF,iCACE,eAAA,EAMJ,0CACE,IAAA,EACA,MAAA,KACA,KAAA,KACA,WAAA,EACA,aAAA,QC7EA,mCACE,QAAA,aACA,YAAA,OACA,eAAA,OACA,QAAA,GAWA,mCACE,QAAA,KAGF,oCACE,QAAA,aACA,aAAA,OACA,eAAA,OACA,QAAA,GA9BN,WAAA,KAAA,MAAA,YACA,aAAA,KAAA,MACA,cAAA,KAAA,MAAA,YAiCE,yCACE,YAAA,EDqDF,oCACE,eAAA,EAON,kBACE,OAAA,EACA,OAAA,MAAA,EACA,SAAA,OACA,WAAA,IAAA,MAAA,gBAMF,eACE,QAAA,MACA,MAAA,KACA,QAAA,OAAA,KACA,MAAA,KACA,YAAA,IACA,MAAA,QACA,WAAA,QACA,gBAAA,KACA,YAAA,OACA,iBAAA,YACA,OAAA,EAcA,qBAAA,qBAEE,MAAA,QVzJF,iBAAA,QU8JA,sBAAA,sBAEE,MAAA,KACA,gBAAA,KVjKF,iBAAA,QUqKA,wBAAA,wBAEE,MAAA,QACA,eAAA,KACA,iBAAA,YAMJ,oBACE,QAAA,MAIF,iBACE,QAAA,MACA,QAAA,MAAA,KACA,cAAA,E3B0GI,UAAA,Q2BxGJ,MAAA,QACA,YAAA,OAIF,oBACE,QAAA,MACA,QAAA,OAAA,KACA,MAAA,QAIF,oBACE,MAAA,QACA,iBAAA,QACA,aAAA,gBAGA,mCACE,MAAA,QAEA,yCAAA,yCAEE,MAAA,KVhNJ,iBAAA,sBUoNE,0CAAA,0CAEE,MAAA,KVtNJ,iBAAA,QU0NE,4CAAA,4CAEE,MAAA,QAIJ,sCACE,aAAA,gBAGF,wCACE,MAAA,QAGF,qCACE,MAAA,QE5OJ,W9B2rHA,oB8BzrHE,SAAA,SACA,QAAA,YACA,eAAA,O9B6rHF,yB8B3rHE,gBACE,SAAA,SACA,KAAA,EAAA,EAAA,K9BmsHJ,4CACA,0CAIA,gCADA,gCADA,+BADA,+B8BhsHE,mC9ByrHF,iCAIA,uBADA,uBADA,sBADA,sB8BprHI,QAAA,EAKJ,aACE,QAAA,KACA,UAAA,KACA,gBAAA,WAEA,0BACE,MAAA,K9BgsHJ,wC8B1rHE,kCAEE,YAAA,K9B4rHJ,4C8BxrHE,uD5BRE,wBAAA,EACA,2BAAA,EFqsHJ,6C8BrrHE,+B9BorHF,iCEvrHI,uBAAA,EACA,0BAAA,E4BqBJ,uBACE,cAAA,SACA,aAAA,SAEA,8BAAA,uCAAA,sCAGE,YAAA,EAGF,0CACE,aAAA,EAIJ,0CAAA,+BACE,cAAA,QACA,aAAA,QAGF,0CAAA,+BACE,cAAA,OACA,aAAA,OAoBF,oBACE,eAAA,OACA,YAAA,WACA,gBAAA,OAEA,yB9BmpHF,+B8BjpHI,MAAA,K9BqpHJ,iD8BlpHE,2CAEE,WAAA,K9BopHJ,qD8BhpHE,gE5BvFE,2BAAA,EACA,0BAAA,EF2uHJ,sD8BhpHE,8B5B1GE,uBAAA,EACA,wBAAA,E6BxBJ,KACE,QAAA,KACA,UAAA,KACA,aAAA,EACA,cAAA,EACA,WAAA,KAGF,UACE,QAAA,MACA,QAAA,MAAA,KAGA,MAAA,QACA,gBAAA,KdHI,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,YAIA,uCcPN,UdQQ,WAAA,McCN,gBAAA,gBAEE,MAAA,QAKF,mBACE,MAAA,QACA,eAAA,KACA,OAAA,QAQJ,UACE,cAAA,IAAA,MAAA,QAEA,oBACE,cAAA,KACA,WAAA,IACA,OAAA,IAAA,MAAA,Y7BlBA,uBAAA,OACA,wBAAA,O6BoBA,0BAAA,0BAEE,aAAA,QAAA,QAAA,QAEA,UAAA,QAGF,6BACE,MAAA,QACA,iBAAA,YACA,aAAA,Y/BixHN,mC+B7wHE,2BAEE,MAAA,QACA,iBAAA,KACA,aAAA,QAAA,QAAA,KAGF,yBAEE,WAAA,K7B5CA,uBAAA,EACA,wBAAA,E6BuDF,qBACE,WAAA,IACA,OAAA,E7BnEA,cAAA,O6BuEF,4B/BmwHF,2B+BjwHI,MAAA,KbxFF,iBAAA,QlB+1HF,oB+B5vHE,oBAEE,KAAA,EAAA,EAAA,KACA,WAAA,O/B+vHJ,yB+B1vHE,yBAEE,WAAA,EACA,UAAA,EACA,WAAA,OAMF,8B/BuvHF,mC+BtvHI,MAAA,KAUF,uBACE,QAAA,KAEF,qBACE,QAAA,MCxHJ,QACE,SAAA,SACA,QAAA,KACA,UAAA,KACA,YAAA,OACA,gBAAA,cACA,YAAA,MAEA,eAAA,MAOA,mBhCs2HF,yBAGA,sBADA,sBADA,sBAGA,sBACA,uBgC12HI,QAAA,KACA,UAAA,QACA,YAAA,OACA,gBAAA,cAoBJ,cACE,YAAA,SACA,eAAA,SACA,aAAA,K/B2OI,UAAA,Q+BzOJ,gBAAA,KACA,YAAA,OAaF,YACE,QAAA,KACA,eAAA,OACA,aAAA,EACA,cAAA,EACA,WAAA,KAEA,sBACE,cAAA,EACA,aAAA,EAGF,2BACE,SAAA,OASJ,aACE,YAAA,MACA,eAAA,MAYF,iBACE,WAAA,KACA,UAAA,EAGA,YAAA,OAIF,gBACE,QAAA,OAAA,O/B6KI,UAAA,Q+B3KJ,YAAA,EACA,iBAAA,YACA,OAAA,IAAA,MAAA,Y9BzGE,cAAA,OeHE,WAAA,WAAA,KAAA,YAIA,uCemGN,gBflGQ,WAAA,Me2GN,sBACE,gBAAA,KAGF,sBACE,gBAAA,KACA,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,OAMJ,qBACE,QAAA,aACA,MAAA,MACA,OAAA,MACA,eAAA,OACA,kBAAA,UACA,oBAAA,OACA,gBAAA,KAGF,mBACE,WAAA,6BACA,WAAA,KvB1FE,yBuBsGA,kBAEI,UAAA,OACA,gBAAA,WAEA,8BACE,eAAA,IAEA,6CACE,SAAA,SAGF,wCACE,cAAA,MACA,aAAA,MAIJ,qCACE,SAAA,QAGF,mCACE,QAAA,eACA,WAAA,KAGF,kCACE,QAAA,KAGF,oCACE,QAAA,KAGF,6BACE,SAAA,QACA,OAAA,EACA,QAAA,KACA,UAAA,EACA,WAAA,kBACA,iBAAA,YACA,aAAA,EACA,YAAA,EfhMJ,WAAA,KekMI,UAAA,KhC+yHV,oCgC7yHQ,iCAEE,OAAA,KACA,WAAA,EACA,cAAA,EAGF,kCACE,QAAA,KACA,UAAA,EACA,QAAA,EACA,WAAA,SvBhKN,yBuBsGA,kBAEI,UAAA,OACA,gBAAA,WAEA,8BACE,eAAA,IAEA,6CACE,SAAA,SAGF,wCACE,cAAA,MACA,aAAA,MAIJ,qCACE,SAAA,QAGF,mCACE,QAAA,eACA,WAAA,KAGF,kCACE,QAAA,KAGF,oCACE,QAAA,KAGF,6BACE,SAAA,QACA,OAAA,EACA,QAAA,KACA,UAAA,EACA,WAAA,kBACA,iBAAA,YACA,aAAA,EACA,YAAA,EfhMJ,WAAA,KekMI,UAAA,KhCo2HV,oCgCl2HQ,iCAEE,OAAA,KACA,WAAA,EACA,cAAA,EAGF,kCACE,QAAA,KACA,UAAA,EACA,QAAA,EACA,WAAA,SvBhKN,yBuBsGA,kBAEI,UAAA,OACA,gBAAA,WAEA,8BACE,eAAA,IAEA,6CACE,SAAA,SAGF,wCACE,cAAA,MACA,aAAA,MAIJ,qCACE,SAAA,QAGF,mCACE,QAAA,eACA,WAAA,KAGF,kCACE,QAAA,KAGF,oCACE,QAAA,KAGF,6BACE,SAAA,QACA,OAAA,EACA,QAAA,KACA,UAAA,EACA,WAAA,kBACA,iBAAA,YACA,aAAA,EACA,YAAA,EfhMJ,WAAA,KekMI,UAAA,KhCy5HV,oCgCv5HQ,iCAEE,OAAA,KACA,WAAA,EACA,cAAA,EAGF,kCACE,QAAA,KACA,UAAA,EACA,QAAA,EACA,WAAA,SvBhKN,0BuBsGA,kBAEI,UAAA,OACA,gBAAA,WAEA,8BACE,eAAA,IAEA,6CACE,SAAA,SAGF,wCACE,cAAA,MACA,aAAA,MAIJ,qCACE,SAAA,QAGF,mCACE,QAAA,eACA,WAAA,KAGF,kCACE,QAAA,KAGF,oCACE,QAAA,KAGF,6BACE,SAAA,QACA,OAAA,EACA,QAAA,KACA,UAAA,EACA,WAAA,kBACA,iBAAA,YACA,aAAA,EACA,YAAA,EfhMJ,WAAA,KekMI,UAAA,KhC88HV,oCgC58HQ,iCAEE,OAAA,KACA,WAAA,EACA,cAAA,EAGF,kCACE,QAAA,KACA,UAAA,EACA,QAAA,EACA,WAAA,SvBhKN,0BuBsGA,mBAEI,UAAA,OACA,gBAAA,WAEA,+BACE,eAAA,IAEA,8CACE,SAAA,SAGF,yCACE,cAAA,MACA,aAAA,MAIJ,sCACE,SAAA,QAGF,oCACE,QAAA,eACA,WAAA,KAGF,mCACE,QAAA,KAGF,qCACE,QAAA,KAGF,8BACE,SAAA,QACA,OAAA,EACA,QAAA,KACA,UAAA,EACA,WAAA,kBACA,iBAAA,YACA,aAAA,EACA,YAAA,EfhMJ,WAAA,KekMI,UAAA,KhCmgIV,qCgCjgIQ,kCAEE,OAAA,KACA,WAAA,EACA,cAAA,EAGF,mCACE,QAAA,KACA,UAAA,EACA,QAAA,EACA,WAAA,SA1DN,eAEI,UAAA,OACA,gBAAA,WAEA,2BACE,eAAA,IAEA,0CACE,SAAA,SAGF,qCACE,cAAA,MACA,aAAA,MAIJ,kCACE,SAAA,QAGF,gCACE,QAAA,eACA,WAAA,KAGF,+BACE,QAAA,KAGF,iCACE,QAAA,KAGF,0BACE,SAAA,QACA,OAAA,EACA,QAAA,KACA,UAAA,EACA,WAAA,kBACA,iBAAA,YACA,aAAA,EACA,YAAA,EfhMJ,WAAA,KekMI,UAAA,KhCujIV,iCgCrjIQ,8BAEE,OAAA,KACA,WAAA,EACA,cAAA,EAGF,+BACE,QAAA,KACA,UAAA,EACA,QAAA,EACA,WAAA,QAcR,4BACE,MAAA,eAEA,kCAAA,kCAEE,MAAA,eAKF,oCACE,MAAA,gBAEA,0CAAA,0CAEE,MAAA,eAGF,6CACE,MAAA,ehCqiIR,2CgCjiII,0CAEE,MAAA,eAIJ,8BACE,MAAA,gBACA,aAAA,eAGF,mCACE,iBAAA,4OAGF,2BACE,MAAA,gBAEA,6BhC8hIJ,mCADA,mCgC1hIM,MAAA,eAOJ,2BACE,MAAA,KAEA,iCAAA,iCAEE,MAAA,KAKF,mCACE,MAAA,sBAEA,yCAAA,yCAEE,MAAA,sBAGF,4CACE,MAAA,sBhCqhIR,0CgCjhII,yCAEE,MAAA,KAIJ,6BACE,MAAA,sBACA,aAAA,qBAGF,kCACE,iBAAA,kPAGF,0BACE,MAAA,sBACA,4BhC+gIJ,kCADA,kCgC3gIM,MAAA,KCvUN,MACE,SAAA,SACA,QAAA,KACA,eAAA,OACA,UAAA,EAEA,UAAA,WACA,iBAAA,KACA,gBAAA,WACA,OAAA,IAAA,MAAA,iB/BME,cAAA,O+BFF,SACE,aAAA,EACA,YAAA,EAGF,kBACE,WAAA,QACA,cAAA,QAEA,8BACE,iBAAA,E/BCF,uBAAA,mBACA,wBAAA,mB+BEA,6BACE,oBAAA,E/BUF,2BAAA,mBACA,0BAAA,mB+BJF,+BjCk1IF,+BiCh1II,WAAA,EAIJ,WAGE,KAAA,EAAA,EAAA,KACA,QAAA,KAAA,KAIF,YACE,cAAA,MAGF,eACE,WAAA,QACA,cAAA,EAGF,sBACE,cAAA,EAQA,sBACE,YAAA,KAQJ,aACE,QAAA,MAAA,KACA,cAAA,EAEA,iBAAA,gBACA,cAAA,IAAA,MAAA,iBAEA,yB/BpEE,cAAA,mBAAA,mBAAA,EAAA,E+ByEJ,aACE,QAAA,MAAA,KAEA,iBAAA,gBACA,WAAA,IAAA,MAAA,iBAEA,wB/B/EE,cAAA,EAAA,EAAA,mBAAA,mB+ByFJ,kBACE,aAAA,OACA,cAAA,OACA,YAAA,OACA,cAAA,EAUF,mBACE,aAAA,OACA,YAAA,OAIF,kBACE,SAAA,SACA,IAAA,EACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,K/BnHE,cAAA,mB+BuHJ,UjCozIA,iBADA,ciChzIE,MAAA,KAGF,UjCmzIA,cEv6II,uBAAA,mBACA,wBAAA,mB+BwHJ,UjCozIA,iBE/5II,2BAAA,mBACA,0BAAA,mB+BuHF,kBACE,cAAA,OxBpGA,yBwBgGJ,YAQI,QAAA,KACA,UAAA,IAAA,KAGA,kBAEE,KAAA,EAAA,EAAA,GACA,cAAA,EAEA,wBACE,YAAA,EACA,YAAA,EAKA,mC/BpJJ,wBAAA,EACA,2BAAA,EF+7IJ,gDiCzyIU,iDAGE,wBAAA,EjC0yIZ,gDiCxyIU,oDAGE,2BAAA,EAIJ,oC/BrJJ,uBAAA,EACA,0BAAA,EF67IJ,iDiCtyIU,kDAGE,uBAAA,EjCuyIZ,iDiCryIU,qDAGE,0BAAA,GC7MZ,kBACE,SAAA,SACA,QAAA,KACA,YAAA,OACA,MAAA,KACA,QAAA,KAAA,QjC4RI,UAAA,KiC1RJ,MAAA,QACA,WAAA,KACA,iBAAA,KACA,OAAA,EhCKE,cAAA,EgCHF,gBAAA,KjBAI,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,WAAA,CAAA,cAAA,KAAA,KAIA,uCiBhBN,kBjBiBQ,WAAA,MiBFN,kCACE,MAAA,QACA,iBAAA,QACA,WAAA,MAAA,EAAA,KAAA,EAAA,iBAEA,yCACE,iBAAA,gRACA,UAAA,gBAKJ,yBACE,YAAA,EACA,MAAA,QACA,OAAA,QACA,YAAA,KACA,QAAA,GACA,iBAAA,gRACA,kBAAA,UACA,gBAAA,QjBvBE,WAAA,UAAA,IAAA,YAIA,uCiBWJ,yBjBVM,WAAA,MiBsBN,wBACE,QAAA,EAGF,wBACE,QAAA,EACA,aAAA,QACA,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,OAAA,qBAIJ,kBACE,cAAA,EAGF,gBACE,iBAAA,KACA,OAAA,IAAA,MAAA,iBAEA,8BhCnCE,uBAAA,OACA,wBAAA,OgCqCA,gDhCtCA,uBAAA,mBACA,wBAAA,mBgC0CF,oCACE,WAAA,EAIF,6BhClCE,2BAAA,OACA,0BAAA,OgCqCE,yDhCtCF,2BAAA,mBACA,0BAAA,mBgC0CA,iDhC3CA,2BAAA,OACA,0BAAA,OgCgDJ,gBACE,QAAA,KAAA,QASA,qCACE,aAAA,EAGF,iCACE,aAAA,EACA,YAAA,EhCxFA,cAAA,EgC2FA,6CAAgB,WAAA,EAChB,4CAAe,cAAA,EAEf,mDhC9FA,cAAA,EiCnBJ,YACE,QAAA,KACA,UAAA,KACA,QAAA,EAAA,EACA,cAAA,KAEA,WAAA,KAOA,kCACE,aAAA,MAEA,0CACE,MAAA,KACA,cAAA,MACA,MAAA,QACA,QAAA,kCAIJ,wBACE,MAAA,QCzBJ,YACE,QAAA,KhCGA,aAAA,EACA,WAAA,KgCAF,WACE,SAAA,SACA,QAAA,MACA,MAAA,QACA,gBAAA,KACA,iBAAA,KACA,OAAA,IAAA,MAAA,QnBKI,WAAA,MAAA,KAAA,WAAA,CAAA,iBAAA,KAAA,WAAA,CAAA,aAAA,KAAA,WAAA,CAAA,WAAA,KAAA,YAIA,uCmBfN,WnBgBQ,WAAA,MmBPN,iBACE,QAAA,EACA,MAAA,QAEA,iBAAA,QACA,aAAA,QAGF,iBACE,QAAA,EACA,MAAA,QACA,iBAAA,QACA,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,OAAA,qBAKF,wCACE,YAAA,KAGF,6BACE,QAAA,EACA,MAAA,KlBlCF,iBAAA,QkBoCE,aAAA,QAGF,+BACE,MAAA,QACA,eAAA,KACA,iBAAA,KACA,aAAA,QC3CF,WACE,QAAA,QAAA,OAOI,kCnCqCJ,uBAAA,OACA,0BAAA,OmChCI,iCnCiBJ,wBAAA,OACA,2BAAA,OmChCF,0BACE,QAAA,OAAA,OpCgSE,UAAA,QoCzRE,iDnCqCJ,uBAAA,MACA,0BAAA,MmChCI,gDnCiBJ,wBAAA,MACA,2BAAA,MmChCF,0BACE,QAAA,OAAA,MpCgSE,UAAA,QoCzRE,iDnCqCJ,uBAAA,MACA,0BAAA,MmChCI,gDnCiBJ,wBAAA,MACA,2BAAA,MoC/BJ,OACE,QAAA,aACA,QAAA,MAAA,MrC8RI,UAAA,MqC5RJ,YAAA,IACA,YAAA,EACA,MAAA,KACA,WAAA,OACA,YAAA,OACA,eAAA,SpCKE,cAAA,OoCAF,aACE,QAAA,KAKJ,YACE,SAAA,SACA,IAAA,KCvBF,OACE,SAAA,SACA,QAAA,KAAA,KACA,cAAA,KACA,OAAA,IAAA,MAAA,YrCWE,cAAA,OqCNJ,eAEE,MAAA,QAIF,YACE,YAAA,IAQF,mBACE,cAAA,KAGA,8BACE,SAAA,SACA,IAAA,EACA,MAAA,EACA,QAAA,EACA,QAAA,QAAA,KAeF,eClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,2BACE,MAAA,QD6CF,iBClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,6BACE,MAAA,QD6CF,eClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,2BACE,MAAA,QD6CF,YClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,wBACE,MAAA,QD6CF,eClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,2BACE,MAAA,QD6CF,cClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,0BACE,MAAA,QD6CF,aClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,yBACE,MAAA,QD6CF,YClDA,MAAA,QtBEA,iBAAA,QsBAA,aAAA,QAEA,wBACE,MAAA,QCHF,wCACE,GAAK,sBAAA,MADP,gCACE,GAAK,sBAAA,MAKT,UACE,QAAA,KACA,OAAA,KACA,SAAA,OxCwRI,UAAA,OwCtRJ,iBAAA,QvCIE,cAAA,OuCCJ,cACE,QAAA,KACA,eAAA,OACA,gBAAA,OACA,SAAA,OACA,MAAA,KACA,WAAA,OACA,YAAA,OACA,iBAAA,QxBZI,WAAA,MAAA,IAAA,KAIA,uCwBAN,cxBCQ,WAAA,MwBWR,sBvBYE,iBAAA,iKuBVA,gBAAA,KAAA,KAIA,uBACE,kBAAA,GAAA,OAAA,SAAA,qBAAA,UAAA,GAAA,OAAA,SAAA,qBAGE,uCAJJ,uBAKM,kBAAA,KAAA,UAAA,MCvCR,YACE,QAAA,KACA,eAAA,OAGA,aAAA,EACA,cAAA,ExCSE,cAAA,OwCLJ,qBACE,gBAAA,KACA,cAAA,QAEA,gCAEE,QAAA,uBAAA,KACA,kBAAA,QAUJ,wBACE,MAAA,KACA,MAAA,QACA,WAAA,QAGA,8BAAA,8BAEE,QAAA,EACA,MAAA,QACA,gBAAA,KACA,iBAAA,QAGF,+BACE,MAAA,QACA,iBAAA,QASJ,iBACE,SAAA,SACA,QAAA,MACA,QAAA,MAAA,KACA,MAAA,QACA,gBAAA,KACA,iBAAA,KACA,OAAA,IAAA,MAAA,iBAEA,6BxCrCE,uBAAA,QACA,wBAAA,QwCwCF,4BxC3BE,2BAAA,QACA,0BAAA,QwC8BF,0BAAA,0BAEE,MAAA,QACA,eAAA,KACA,iBAAA,KAIF,wBACE,QAAA,EACA,MAAA,KACA,iBAAA,QACA,aAAA,QAGF,kCACE,iBAAA,EAEA,yCACE,WAAA,KACA,iBAAA,IAcF,uBACE,eAAA,IAGE,oDxCrCJ,0BAAA,OAZA,wBAAA,EwCsDI,mDxCtDJ,wBAAA,OAYA,0BAAA,EwC+CI,+CACE,WAAA,EAGF,yDACE,iBAAA,IACA,kBAAA,EAEA,gEACE,YAAA,KACA,kBAAA,IjCpER,yBiC4CA,0BACE,eAAA,IAGE,uDxCrCJ,0BAAA,OAZA,wBAAA,EwCsDI,sDxCtDJ,wBAAA,OAYA,0BAAA,EwC+CI,kDACE,WAAA,EAGF,4DACE,iBAAA,IACA,kBAAA,EAEA,mEACE,YAAA,KACA,kBAAA,KjCpER,yBiC4CA,0BACE,eAAA,IAGE,uDxCrCJ,0BAAA,OAZA,wBAAA,EwCsDI,sDxCtDJ,wBAAA,OAYA,0BAAA,EwC+CI,kDACE,WAAA,EAGF,4DACE,iBAAA,IACA,kBAAA,EAEA,mEACE,YAAA,KACA,kBAAA,KjCpER,yBiC4CA,0BACE,eAAA,IAGE,uDxCrCJ,0BAAA,OAZA,wBAAA,EwCsDI,sDxCtDJ,wBAAA,OAYA,0BAAA,EwC+CI,kDACE,WAAA,EAGF,4DACE,iBAAA,IACA,kBAAA,EAEA,mEACE,YAAA,KACA,kBAAA,KjCpER,0BiC4CA,0BACE,eAAA,IAGE,uDxCrCJ,0BAAA,OAZA,wBAAA,EwCsDI,sDxCtDJ,wBAAA,OAYA,0BAAA,EwC+CI,kDACE,WAAA,EAGF,4DACE,iBAAA,IACA,kBAAA,EAEA,mEACE,YAAA,KACA,kBAAA,KjCpER,0BiC4CA,2BACE,eAAA,IAGE,wDxCrCJ,0BAAA,OAZA,wBAAA,EwCsDI,uDxCtDJ,wBAAA,OAYA,0BAAA,EwC+CI,mDACE,WAAA,EAGF,6DACE,iBAAA,IACA,kBAAA,EAEA,oEACE,YAAA,KACA,kBAAA,KAcZ,kBxC9HI,cAAA,EwCiIF,mCACE,aAAA,EAAA,EAAA,IAEA,8CACE,oBAAA,ECpJJ,yBACE,MAAA,QACA,iBAAA,QAGE,sDAAA,sDAEE,MAAA,QACA,iBAAA,QAGF,uDACE,MAAA,KACA,iBAAA,QACA,aAAA,QAdN,2BACE,MAAA,QACA,iBAAA,QAGE,wDAAA,wDAEE,MAAA,QACA,iBAAA,QAGF,yDACE,MAAA,KACA,iBAAA,QACA,aAAA,QAdN,yBACE,MAAA,QACA,iBAAA,QAGE,sDAAA,sDAEE,MAAA,QACA,iBAAA,QAGF,uDACE,MAAA,KACA,iBAAA,QACA,aAAA,QAdN,sBACE,MAAA,QACA,iBAAA,QAGE,mDAAA,mDAEE,MAAA,QACA,iBAAA,QAGF,oDACE,MAAA,KACA,iBAAA,QACA,aAAA,QAdN,yBACE,MAAA,QACA,iBAAA,QAGE,sDAAA,sDAEE,MAAA,QACA,iBAAA,QAGF,uDACE,MAAA,KACA,iBAAA,QACA,aAAA,QAdN,wBACE,MAAA,QACA,iBAAA,QAGE,qDAAA,qDAEE,MAAA,QACA,iBAAA,QAGF,sDACE,MAAA,KACA,iBAAA,QACA,aAAA,QAdN,uBACE,MAAA,QACA,iBAAA,QAGE,oDAAA,oDAEE,MAAA,QACA,iBAAA,QAGF,qDACE,MAAA,KACA,iBAAA,QACA,aAAA,QAdN,sBACE,MAAA,QACA,iBAAA,QAGE,mDAAA,mDAEE,MAAA,QACA,iBAAA,QAGF,oDACE,MAAA,KACA,iBAAA,QACA,aAAA,QCbR,WACE,WAAA,YACA,MAAA,IACA,OAAA,IACA,QAAA,MAAA,MACA,MAAA,KACA,WAAA,YAAA,0TAAA,MAAA,CAAA,IAAA,KAAA,UACA,OAAA,E1COE,cAAA,O0CLF,QAAA,GAGA,iBACE,MAAA,KACA,gBAAA,KACA,QAAA,IAGF,iBACE,QAAA,EACA,WAAA,EAAA,EAAA,EAAA,OAAA,qBACA,QAAA,EAGF,oBAAA,oBAEE,eAAA,KACA,oBAAA,KAAA,iBAAA,KAAA,YAAA,KACA,QAAA,IAIJ,iBACE,OAAA,UAAA,gBAAA,iBCtCF,OACE,MAAA,MACA,UAAA,K5CmSI,UAAA,Q4ChSJ,eAAA,KACA,iBAAA,sBACA,gBAAA,YACA,OAAA,IAAA,MAAA,eACA,WAAA,EAAA,MAAA,KAAA,gB3CUE,cAAA,O2CPF,eACE,QAAA,EAGF,kBACE,QAAA,KAIJ,iBACE,MAAA,oBAAA,MAAA,iBAAA,MAAA,YACA,UAAA,KACA,eAAA,KAEA,mCACE,cAAA,OAIJ,cACE,QAAA,KACA,YAAA,OACA,QAAA,MAAA,OACA,MAAA,QACA,iBAAA,sBACA,gBAAA,YACA,cAAA,IAAA,MAAA,gB3CVE,uBAAA,mBACA,wBAAA,mB2CYF,yBACE,aAAA,SACA,YAAA,OAIJ,YACE,QAAA,OACA,UAAA,WC1CF,OACE,SAAA,MACA,IAAA,EACA,KAAA,EACA,QAAA,KACA,QAAA,KACA,MAAA,KACA,OAAA,KACA,WAAA,OACA,WAAA,KAGA,QAAA,EAOF,cACE,SAAA,SACA,MAAA,KACA,OAAA,MAEA,eAAA,KAGA,0B7BlBI,WAAA,UAAA,IAAA,S6BoBF,UAAA,mB7BhBE,uC6BcJ,0B7BbM,WAAA,M6BiBN,0BACE,UAAA,KAIF,kCACE,UAAA,YAIJ,yBACE,OAAA,kBAEA,wCACE,WAAA,KACA,SAAA,OAGF,qCACE,WAAA,KAIJ,uBACE,QAAA,KACA,YAAA,OACA,WAAA,kBAIF,eACE,SAAA,SACA,QAAA,KACA,eAAA,OACA,MAAA,KAGA,eAAA,KACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,e5C3DE,cAAA,M4C+DF,QAAA,EAIF,gBCpFE,SAAA,MACA,IAAA,EACA,KAAA,EACA,QAAA,KACA,MAAA,MACA,OAAA,MACA,iBAAA,KAGA,qBAAS,QAAA,EACT,qBAAS,QAAA,GDgFX,cACE,QAAA,KACA,YAAA,EACA,YAAA,OACA,gBAAA,cACA,QAAA,KAAA,KACA,cAAA,IAAA,MAAA,Q5CtEE,uBAAA,kBACA,wBAAA,kB4CwEF,yBACE,QAAA,MAAA,MACA,OAAA,OAAA,OAAA,OAAA,KAKJ,aACE,cAAA,EACA,YAAA,IAKF,YACE,SAAA,SAGA,KAAA,EAAA,EAAA,KACA,QAAA,KAIF,cACE,QAAA,KACA,UAAA,KACA,YAAA,EACA,YAAA,OACA,gBAAA,SACA,QAAA,OACA,WAAA,IAAA,MAAA,Q5CzFE,2BAAA,kBACA,0BAAA,kB4C8FF,gBACE,OAAA,OrC3EA,yBqCkFF,cACE,UAAA,MACA,OAAA,QAAA,KAGF,yBACE,OAAA,oBAGF,uBACE,WAAA,oBAOF,UAAY,UAAA,OrCnGV,yBqCuGF,U9CywKF,U8CvwKI,UAAA,OrCzGA,0BqC8GF,UAAY,UAAA,QASV,kBACE,MAAA,MACA,UAAA,KACA,OAAA,KACA,OAAA,EAEA,iCACE,OAAA,KACA,OAAA,E5C3KJ,cAAA,E4C+KE,gC5C/KF,cAAA,E4CmLE,8BACE,WAAA,KAGF,gC5CvLF,cAAA,EOyDA,4BqC0GA,0BACE,MAAA,MACA,UAAA,KACA,OAAA,KACA,OAAA,EAEA,yCACE,OAAA,KACA,OAAA,E5C3KJ,cAAA,E4C+KE,wC5C/KF,cAAA,E4CmLE,sCACE,WAAA,KAGF,wC5CvLF,cAAA,GOyDA,4BqC0GA,0BACE,MAAA,MACA,UAAA,KACA,OAAA,KACA,OAAA,EAEA,yCACE,OAAA,KACA,OAAA,E5C3KJ,cAAA,E4C+KE,wC5C/KF,cAAA,E4CmLE,sCACE,WAAA,KAGF,wC5CvLF,cAAA,GOyDA,4BqC0GA,0BACE,MAAA,MACA,UAAA,KACA,OAAA,KACA,OAAA,EAEA,yCACE,OAAA,KACA,OAAA,E5C3KJ,cAAA,E4C+KE,wC5C/KF,cAAA,E4CmLE,sCACE,WAAA,KAGF,wC5CvLF,cAAA,GOyDA,6BqC0GA,0BACE,MAAA,MACA,UAAA,KACA,OAAA,KACA,OAAA,EAEA,yCACE,OAAA,KACA,OAAA,E5C3KJ,cAAA,E4C+KE,wC5C/KF,cAAA,E4CmLE,sCACE,WAAA,KAGF,wC5CvLF,cAAA,GOyDA,6BqC0GA,2BACE,MAAA,MACA,UAAA,KACA,OAAA,KACA,OAAA,EAEA,0CACE,OAAA,KACA,OAAA,E5C3KJ,cAAA,E4C+KE,yC5C/KF,cAAA,E4CmLE,uCACE,WAAA,KAGF,yC5CvLF,cAAA,G8ClBJ,SACE,SAAA,SACA,QAAA,KACA,QAAA,MACA,OAAA,ECJA,YAAA,0BAEA,WAAA,OACA,YAAA,IACA,YAAA,IACA,WAAA,KACA,WAAA,MACA,gBAAA,KACA,YAAA,KACA,eAAA,KACA,eAAA,OACA,WAAA,OACA,aAAA,OACA,YAAA,OACA,WAAA,KhDsRI,UAAA,Q+C1RJ,UAAA,WACA,QAAA,EAEA,cAAS,QAAA,GAET,wBACE,SAAA,SACA,QAAA,MACA,MAAA,MACA,OAAA,MAEA,gCACE,SAAA,SACA,QAAA,GACA,aAAA,YACA,aAAA,MAKN,6CAAA,gBACE,QAAA,MAAA,EAEA,4DAAA,+BACE,OAAA,EAEA,oEAAA,uCACE,IAAA,KACA,aAAA,MAAA,MAAA,EACA,iBAAA,KAKN,+CAAA,gBACE,QAAA,EAAA,MAEA,8DAAA,+BACE,KAAA,EACA,MAAA,MACA,OAAA,MAEA,sEAAA,uCACE,MAAA,KACA,aAAA,MAAA,MAAA,MAAA,EACA,mBAAA,KAKN,gDAAA,mBACE,QAAA,MAAA,EAEA,+DAAA,kCACE,IAAA,EAEA,uEAAA,0CACE,OAAA,KACA,aAAA,EAAA,MAAA,MACA,oBAAA,KAKN,8CAAA,kBACE,QAAA,EAAA,MAEA,6DAAA,iCACE,MAAA,EACA,MAAA,MACA,OAAA,MAEA,qEAAA,yCACE,KAAA,KACA,aAAA,MAAA,EAAA,MAAA,MACA,kBAAA,KAqBN,eACE,UAAA,MACA,QAAA,OAAA,MACA,MAAA,KACA,WAAA,OACA,iBAAA,K9C7FE,cAAA,OgDnBJ,SACE,SAAA,SACA,IAAA,EACA,KAAA,EACA,QAAA,KACA,QAAA,MACA,UAAA,MDLA,YAAA,0BAEA,WAAA,OACA,YAAA,IACA,YAAA,IACA,WAAA,KACA,WAAA,MACA,gBAAA,KACA,YAAA,KACA,eAAA,KACA,eAAA,OACA,WAAA,OACA,aAAA,OACA,YAAA,OACA,WAAA,KhDsRI,UAAA,QiDzRJ,UAAA,WACA,iBAAA,KACA,gBAAA,YACA,OAAA,IAAA,MAAA,ehDIE,cAAA,MgDAF,wBACE,SAAA,SACA,QAAA,MACA,MAAA,KACA,OAAA,MAEA,+BAAA,gCAEE,SAAA,SACA,QAAA,MACA,QAAA,GACA,aAAA,YACA,aAAA,MAMJ,4DAAA,+BACE,OAAA,mBAEA,oEAAA,uCACE,OAAA,EACA,aAAA,MAAA,MAAA,EACA,iBAAA,gBAGF,mEAAA,sCACE,OAAA,IACA,aAAA,MAAA,MAAA,EACA,iBAAA,KAMJ,8DAAA,+BACE,KAAA,mBACA,MAAA,MACA,OAAA,KAEA,sEAAA,uCACE,KAAA,EACA,aAAA,MAAA,MAAA,MAAA,EACA,mBAAA,gBAGF,qEAAA,sCACE,KAAA,IACA,aAAA,MAAA,MAAA,MAAA,EACA,mBAAA,KAMJ,+DAAA,kCACE,IAAA,mBAEA,uEAAA,0CACE,IAAA,EACA,aAAA,EAAA,MAAA,MAAA,MACA,oBAAA,gBAGF,sEAAA,yCACE,IAAA,IACA,aAAA,EAAA,MAAA,MAAA,MACA,oBAAA,KAKJ,wEAAA,2CACE,SAAA,SACA,IAAA,EACA,KAAA,IACA,QAAA,MACA,MAAA,KACA,YAAA,OACA,QAAA,GACA,cAAA,IAAA,MAAA,QAKF,6DAAA,iCACE,MAAA,mBACA,MAAA,MACA,OAAA,KAEA,qEAAA,yCACE,MAAA,EACA,aAAA,MAAA,EAAA,MAAA,MACA,kBAAA,gBAGF,oEAAA,wCACE,MAAA,IACA,aAAA,MAAA,EAAA,MAAA,MACA,kBAAA,KAqBN,gBACE,QAAA,MAAA,KACA,cAAA,EjDuJI,UAAA,KiDpJJ,iBAAA,QACA,cAAA,IAAA,MAAA,ehDtHE,uBAAA,kBACA,wBAAA,kBgDwHF,sBACE,QAAA,KAIJ,cACE,QAAA,KAAA,KACA,MAAA,QC/IF,UACE,SAAA,SAGF,wBACE,aAAA,MAGF,gBACE,SAAA,SACA,MAAA,KACA,SAAA,OCtBA,uBACE,QAAA,MACA,MAAA,KACA,QAAA,GDuBJ,eACE,SAAA,SACA,QAAA,KACA,MAAA,KACA,MAAA,KACA,aAAA,MACA,4BAAA,OAAA,oBAAA,OlClBI,WAAA,UAAA,IAAA,YAIA,uCkCQN,elCPQ,WAAA,MjBgzLR,oBACA,oBmDhyLA,sBAGE,QAAA,MnDmyLF,0BmD/xLA,8CAEE,UAAA,iBnDkyLF,4BmD/xLA,4CAEE,UAAA,kBAWA,8BACE,QAAA,EACA,oBAAA,QACA,UAAA,KnD0xLJ,uDACA,qDmDxxLE,qCAGE,QAAA,EACA,QAAA,EnDyxLJ,yCmDtxLE,2CAEE,QAAA,EACA,QAAA,ElC/DE,WAAA,QAAA,GAAA,IAIA,uCjBq1LN,yCmD7xLE,2ClCvDM,WAAA,MjB01LR,uBmDtxLA,uBAEE,SAAA,SACA,IAAA,EACA,OAAA,EACA,QAAA,EAEA,QAAA,KACA,YAAA,OACA,gBAAA,OACA,MAAA,IACA,QAAA,EACA,MAAA,KACA,WAAA,OACA,WAAA,IACA,OAAA,EACA,QAAA,GlCzFI,WAAA,QAAA,KAAA,KAIA,uCjB82LN,uBmDzyLA,uBlCpEQ,WAAA,MjBm3LR,6BADA,6BmD1xLE,6BAAA,6BAEE,MAAA,KACA,gBAAA,KACA,QAAA,EACA,QAAA,GAGJ,uBACE,KAAA,EAGF,uBACE,MAAA,EnD8xLF,4BmDzxLA,4BAEE,QAAA,aACA,MAAA,KACA,OAAA,KACA,kBAAA,UACA,oBAAA,IACA,gBAAA,KAAA,KAWF,4BACE,iBAAA,wPAEF,4BACE,iBAAA,yPAQF,qBACE,SAAA,SACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,EACA,QAAA,KACA,gBAAA,OACA,QAAA,EAEA,aAAA,IACA,cAAA,KACA,YAAA,IACA,WAAA,KAEA,sCACE,WAAA,YACA,KAAA,EAAA,EAAA,KACA,MAAA,KACA,OAAA,IACA,QAAA,EACA,aAAA,IACA,YAAA,IACA,YAAA,OACA,OAAA,QACA,iBAAA,KACA,gBAAA,YACA,OAAA,EAEA,WAAA,KAAA,MAAA,YACA,cAAA,KAAA,MAAA,YACA,QAAA,GlC5KE,WAAA,QAAA,IAAA,KAIA,uCkCwJJ,sClCvJM,WAAA,MkC2KN,6BACE,QAAA,EASJ,kBACE,SAAA,SACA,MAAA,IACA,OAAA,QACA,KAAA,IACA,YAAA,QACA,eAAA,QACA,MAAA,KACA,WAAA,OnDoxLF,2CmD9wLE,2CAEE,OAAA,UAAA,eAGF,qDACE,iBAAA,KAGF,iCACE,MAAA,KE7NJ,kCACE,GAAK,UAAA,gBADP,0BACE,GAAK,UAAA,gBAIP,gBACE,QAAA,aACA,MAAA,KACA,OAAA,KACA,eAAA,QACA,OAAA,MAAA,MAAA,aACA,mBAAA,YAEA,cAAA,IACA,kBAAA,KAAA,OAAA,SAAA,eAAA,UAAA,KAAA,OAAA,SAAA,eAGF,mBACE,MAAA,KACA,OAAA,KACA,aAAA,KAQF,gCACE,GACE,UAAA,SAEF,IACE,QAAA,EACA,UAAA,MANJ,wBACE,GACE,UAAA,SAEF,IACE,QAAA,EACA,UAAA,MAKJ,cACE,QAAA,aACA,MAAA,KACA,OAAA,KACA,eAAA,QACA,iBAAA,aAEA,cAAA,IACA,QAAA,EACA,kBAAA,KAAA,OAAA,SAAA,aAAA,UAAA,KAAA,OAAA,SAAA,aAGF,iBACE,MAAA,KACA,OAAA,KAIA,uCACE,gBrDo/LJ,cqDl/LM,2BAAA,KAAA,mBAAA,MCjEN,WACE,SAAA,MACA,OAAA,EACA,QAAA,KACA,QAAA,KACA,eAAA,OACA,UAAA,KAEA,WAAA,OACA,iBAAA,KACA,gBAAA,YACA,QAAA,ErCKI,WAAA,UAAA,IAAA,YAIA,uCqCpBN,WrCqBQ,WAAA,MqCLR,oBPdE,SAAA,MACA,IAAA,EACA,KAAA,EACA,QAAA,KACA,MAAA,MACA,OAAA,MACA,iBAAA,KAGA,yBAAS,QAAA,EACT,yBAAS,QAAA,GOQX,kBACE,QAAA,KACA,YAAA,OACA,gBAAA,cACA,QAAA,KAAA,KAEA,6BACE,QAAA,MAAA,MACA,WAAA,OACA,aAAA,OACA,cAAA,OAIJ,iBACE,cAAA,EACA,YAAA,IAGF,gBACE,UAAA,EACA,QAAA,KAAA,KACA,WAAA,KAGF,iBACE,IAAA,EACA,KAAA,EACA,MAAA,MACA,aAAA,IAAA,MAAA,eACA,UAAA,kBAGF,eACE,IAAA,EACA,MAAA,EACA,MAAA,MACA,YAAA,IAAA,MAAA,eACA,UAAA,iBAGF,eACE,IAAA,EACA,MAAA,EACA,KAAA,EACA,OAAA,KACA,WAAA,KACA,cAAA,IAAA,MAAA,eACA,UAAA,kBAGF,kBACE,MAAA,EACA,KAAA,EACA,OAAA,KACA,WAAA,KACA,WAAA,IAAA,MAAA,eACA,UAAA,iBAGF,gBACE,UAAA,KCjFF,aACE,QAAA,aACA,WAAA,IACA,eAAA,OACA,OAAA,KACA,iBAAA,aACA,QAAA,GAEA,yBACE,QAAA,aACA,QAAA,GAKJ,gBACE,WAAA,KAGF,gBACE,WAAA,KAGF,gBACE,WAAA,MAKA,+BACE,kBAAA,iBAAA,GAAA,YAAA,SAAA,UAAA,iBAAA,GAAA,YAAA,SAIJ,oCACE,IACE,QAAA,IAFJ,4BACE,IACE,QAAA,IAIJ,kBACE,mBAAA,8DAAA,WAAA,8DACA,kBAAA,KAAA,KAAA,UAAA,KAAA,KACA,kBAAA,iBAAA,GAAA,OAAA,SAAA,UAAA,iBAAA,GAAA,OAAA,SAGF,oCACE,KACE,sBAAA,MAAA,GAAA,cAAA,MAAA,IAFJ,4BACE,KACE,sBAAA,MAAA,GAAA,cAAA,MAAA,IH9CF,iBACE,QAAA,MACA,MAAA,KACA,QAAA,GIJF,cACE,MAAA,QAGE,oBAAA,oBAEE,MAAA,QANN,gBACE,MAAA,QAGE,sBAAA,sBAEE,MAAA,QANN,cACE,MAAA,QAGE,oBAAA,oBAEE,MAAA,QANN,WACE,MAAA,QAGE,iBAAA,iBAEE,MAAA,QANN,cACE,MAAA,QAGE,oBAAA,oBAEE,MAAA,QANN,aACE,MAAA,QAGE,mBAAA,mBAEE,MAAA,QANN,YACE,MAAA,QAGE,kBAAA,kBAEE,MAAA,QANN,WACE,MAAA,QAGE,iBAAA,iBAEE,MAAA,QCLR,OACE,SAAA,SACA,MAAA,KAEA,eACE,QAAA,MACA,YAAA,uBACA,QAAA,GAGF,SACE,SAAA,SACA,IAAA,EACA,KAAA,EACA,MAAA,KACA,OAAA,KAKF,WACE,kBAAA,KADF,WACE,kBAAA,mBADF,YACE,kBAAA,oBADF,YACE,kBAAA,oBCrBJ,WACE,SAAA,MACA,IAAA,EACA,MAAA,EACA,KAAA,EACA,QAAA,KAGF,cACE,SAAA,MACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,KAQE,YACE,SAAA,eAAA,SAAA,OACA,IAAA,EACA,QAAA,KjDqCF,yBiDxCA,eACE,SAAA,eAAA,SAAA,OACA,IAAA,EACA,QAAA,MjDqCF,yBiDxCA,eACE,SAAA,eAAA,SAAA,OACA,IAAA,EACA,QAAA,MjDqCF,yBiDxCA,eACE,SAAA,eAAA,SAAA,OACA,IAAA,EACA,QAAA,MjDqCF,0BiDxCA,eACE,SAAA,eAAA,SAAA,OACA,IAAA,EACA,QAAA,MjDqCF,0BiDxCA,gBACE,SAAA,eAAA,SAAA,OACA,IAAA,EACA,QAAA,MCzBN,QACE,QAAA,KACA,eAAA,IACA,YAAA,OACA,WAAA,QAGF,QACE,QAAA,KACA,KAAA,EAAA,EAAA,KACA,eAAA,OACA,WAAA,QCRF,iB5Dk4MA,0D6D93ME,SAAA,mBACA,MAAA,cACA,OAAA,cACA,QAAA,YACA,OAAA,eACA,SAAA,iBACA,KAAA,wBACA,YAAA,iBACA,OAAA,YCXA,uBACE,SAAA,SACA,IAAA,EACA,MAAA,EACA,OAAA,EACA,KAAA,EACA,QAAA,EACA,QAAA,GCRJ,eCAE,SAAA,OACA,cAAA,SACA,YAAA,OCNF,IACE,QAAA,aACA,WAAA,QACA,MAAA,IACA,WAAA,IACA,iBAAA,aACA,QAAA,ICyDM,gBAOI,eAAA,mBAPJ,WAOI,eAAA,cAPJ,cAOI,eAAA,iBAPJ,cAOI,eAAA,iBAPJ,mBAOI,eAAA,sBAPJ,gBAOI,eAAA,mBAPJ,aAOI,MAAA,eAPJ,WAOI,MAAA,gBAPJ,YAOI,MAAA,eAPJ,WAOI,QAAA,YAPJ,YAOI,QAAA,cAPJ,YAOI,QAAA,aAPJ,YAOI,QAAA,cAPJ,aAOI,QAAA,YAPJ,eAOI,SAAA,eAPJ,iBAOI,SAAA,iBAPJ,kBAOI,SAAA,kBAPJ,iBAOI,SAAA,iBAPJ,UAOI,QAAA,iBAPJ,gBAOI,QAAA,uBAPJ,SAOI,QAAA,gBAPJ,QAOI,QAAA,eAPJ,SAOI,QAAA,gBAPJ,aAOI,QAAA,oBAPJ,cAOI,QAAA,qBAPJ,QAOI,QAAA,eAPJ,eAOI,QAAA,sBAPJ,QAOI,QAAA,eAPJ,QAOI,WAAA,EAAA,MAAA,KAAA,0BAPJ,WAOI,WAAA,EAAA,QAAA,OAAA,2BAPJ,WAOI,WAAA,EAAA,KAAA,KAAA,2BAPJ,aAOI,WAAA,eAPJ,iBAOI,SAAA,iBAPJ,mBAOI,SAAA,mBAPJ,mBAOI,SAAA,mBAPJ,gBAOI,SAAA,gBAPJ,iBAOI,SAAA,yBAAA,SAAA,iBAPJ,OAOI,IAAA,YAPJ,QAOI,IAAA,cAPJ,SAOI,IAAA,eAPJ,UAOI,OAAA,YAPJ,WAOI,OAAA,cAPJ,YAOI,OAAA,eAPJ,SAOI,KAAA,YAPJ,UAOI,KAAA,cAPJ,WAOI,KAAA,eAPJ,OAOI,MAAA,YAPJ,QAOI,MAAA,cAPJ,SAOI,MAAA,eAPJ,kBAOI,UAAA,+BAPJ,oBAOI,UAAA,2BAPJ,oBAOI,UAAA,2BAPJ,QAOI,OAAA,IAAA,MAAA,kBAPJ,UAOI,OAAA,YAPJ,YAOI,WAAA,IAAA,MAAA,kBAPJ,cAOI,WAAA,YAPJ,YAOI,aAAA,IAAA,MAAA,kBAPJ,cAOI,aAAA,YAPJ,eAOI,cAAA,IAAA,MAAA,kBAPJ,iBAOI,cAAA,YAPJ,cAOI,YAAA,IAAA,MAAA,kBAPJ,gBAOI,YAAA,YAPJ,gBAOI,aAAA,kBAPJ,kBAOI,aAAA,kBAPJ,gBAOI,aAAA,kBAPJ,aAOI,aAAA,kBAPJ,gBAOI,aAAA,kBAPJ,eAOI,aAAA,kBAPJ,cAOI,aAAA,kBAPJ,aAOI,aAAA,kBAPJ,cAOI,aAAA,eAPJ,UAOI,aAAA,cAPJ,UAOI,aAAA,cAPJ,UAOI,aAAA,cAPJ,UAOI,aAAA,cAPJ,UAOI,aAAA,cAPJ,MAOI,MAAA,cAPJ,MAOI,MAAA,cAPJ,MAOI,MAAA,cAPJ,OAOI,MAAA,eAPJ,QAOI,MAAA,eAPJ,QAOI,UAAA,eAPJ,QAOI,MAAA,gBAPJ,YAOI,UAAA,gBAPJ,MAOI,OAAA,cAPJ,MAOI,OAAA,cAPJ,MAOI,OAAA,cAPJ,OAOI,OAAA,eAPJ,QAOI,OAAA,eAPJ,QAOI,WAAA,eAPJ,QAOI,OAAA,gBAPJ,YAOI,WAAA,gBAPJ,WAOI,KAAA,EAAA,EAAA,eAPJ,UAOI,eAAA,cAPJ,aAOI,eAAA,iBAPJ,kBAOI,eAAA,sBAPJ,qBAOI,eAAA,yBAPJ,aAOI,UAAA,YAPJ,aAOI,UAAA,YAPJ,eAOI,YAAA,YAPJ,eAOI,YAAA,YAPJ,WAOI,UAAA,eAPJ,aAOI,UAAA,iBAPJ,mBAOI,UAAA,uBAPJ,OAOI,IAAA,YAPJ,OAOI,IAAA,iBAPJ,OAOI,IAAA,gBAPJ,OAOI,IAAA,eAPJ,OAOI,IAAA,iBAPJ,OAOI,IAAA,eAPJ,uBAOI,gBAAA,qBAPJ,qBAOI,gBAAA,mBAPJ,wBAOI,gBAAA,iBAPJ,yBAOI,gBAAA,wBAPJ,wBAOI,gBAAA,uBAPJ,wBAOI,gBAAA,uBAPJ,mBAOI,YAAA,qBAPJ,iBAOI,YAAA,mBAPJ,oBAOI,YAAA,iBAPJ,sBAOI,YAAA,mBAPJ,qBAOI,YAAA,kBAPJ,qBAOI,cAAA,qBAPJ,mBAOI,cAAA,mBAPJ,sBAOI,cAAA,iBAPJ,uBAOI,cAAA,wBAPJ,sBAOI,cAAA,uBAPJ,uBAOI,cAAA,kBAPJ,iBAOI,WAAA,eAPJ,kBAOI,WAAA,qBAPJ,gBAOI,WAAA,mBAPJ,mBAOI,WAAA,iBAPJ,qBAOI,WAAA,mBAPJ,oBAOI,WAAA,kBAPJ,aAOI,MAAA,aAPJ,SAOI,MAAA,YAPJ,SAOI,MAAA,YAPJ,SAOI,MAAA,YAPJ,SAOI,MAAA,YAPJ,SAOI,MAAA,YAPJ,SAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,KAOI,OAAA,YAPJ,KAOI,OAAA,iBAPJ,KAOI,OAAA,gBAPJ,KAOI,OAAA,eAPJ,KAOI,OAAA,iBAPJ,KAOI,OAAA,eAPJ,QAOI,OAAA,eAPJ,MAOI,aAAA,YAAA,YAAA,YAPJ,MAOI,aAAA,iBAAA,YAAA,iBAPJ,MAOI,aAAA,gBAAA,YAAA,gBAPJ,MAOI,aAAA,eAAA,YAAA,eAPJ,MAOI,aAAA,iBAAA,YAAA,iBAPJ,MAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,MAOI,WAAA,YAAA,cAAA,YAPJ,MAOI,WAAA,iBAAA,cAAA,iBAPJ,MAOI,WAAA,gBAAA,cAAA,gBAPJ,MAOI,WAAA,eAAA,cAAA,eAPJ,MAOI,WAAA,iBAAA,cAAA,iBAPJ,MAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,MAOI,WAAA,YAPJ,MAOI,WAAA,iBAPJ,MAOI,WAAA,gBAPJ,MAOI,WAAA,eAPJ,MAOI,WAAA,iBAPJ,MAOI,WAAA,eAPJ,SAOI,WAAA,eAPJ,MAOI,aAAA,YAPJ,MAOI,aAAA,iBAPJ,MAOI,aAAA,gBAPJ,MAOI,aAAA,eAPJ,MAOI,aAAA,iBAPJ,MAOI,aAAA,eAPJ,SAOI,aAAA,eAPJ,MAOI,cAAA,YAPJ,MAOI,cAAA,iBAPJ,MAOI,cAAA,gBAPJ,MAOI,cAAA,eAPJ,MAOI,cAAA,iBAPJ,MAOI,cAAA,eAPJ,SAOI,cAAA,eAPJ,MAOI,YAAA,YAPJ,MAOI,YAAA,iBAPJ,MAOI,YAAA,gBAPJ,MAOI,YAAA,eAPJ,MAOI,YAAA,iBAPJ,MAOI,YAAA,eAPJ,SAOI,YAAA,eAPJ,KAOI,QAAA,YAPJ,KAOI,QAAA,iBAPJ,KAOI,QAAA,gBAPJ,KAOI,QAAA,eAPJ,KAOI,QAAA,iBAPJ,KAOI,QAAA,eAPJ,MAOI,cAAA,YAAA,aAAA,YAPJ,MAOI,cAAA,iBAAA,aAAA,iBAPJ,MAOI,cAAA,gBAAA,aAAA,gBAPJ,MAOI,cAAA,eAAA,aAAA,eAPJ,MAOI,cAAA,iBAAA,aAAA,iBAPJ,MAOI,cAAA,eAAA,aAAA,eAPJ,MAOI,YAAA,YAAA,eAAA,YAPJ,MAOI,YAAA,iBAAA,eAAA,iBAPJ,MAOI,YAAA,gBAAA,eAAA,gBAPJ,MAOI,YAAA,eAAA,eAAA,eAPJ,MAOI,YAAA,iBAAA,eAAA,iBAPJ,MAOI,YAAA,eAAA,eAAA,eAPJ,MAOI,YAAA,YAPJ,MAOI,YAAA,iBAPJ,MAOI,YAAA,gBAPJ,MAOI,YAAA,eAPJ,MAOI,YAAA,iBAPJ,MAOI,YAAA,eAPJ,MAOI,cAAA,YAPJ,MAOI,cAAA,iBAPJ,MAOI,cAAA,gBAPJ,MAOI,cAAA,eAPJ,MAOI,cAAA,iBAPJ,MAOI,cAAA,eAPJ,MAOI,eAAA,YAPJ,MAOI,eAAA,iBAPJ,MAOI,eAAA,gBAPJ,MAOI,eAAA,eAPJ,MAOI,eAAA,iBAPJ,MAOI,eAAA,eAPJ,MAOI,aAAA,YAPJ,MAOI,aAAA,iBAPJ,MAOI,aAAA,gBAPJ,MAOI,aAAA,eAPJ,MAOI,aAAA,iBAPJ,MAOI,aAAA,eAPJ,gBAOI,YAAA,mCAPJ,MAOI,UAAA,iCAPJ,MAOI,UAAA,gCAPJ,MAOI,UAAA,8BAPJ,MAOI,UAAA,gCAPJ,MAOI,UAAA,kBAPJ,MAOI,UAAA,eAPJ,YAOI,WAAA,iBAPJ,YAOI,WAAA,iBAPJ,UAOI,YAAA,cAPJ,YAOI,YAAA,kBAPJ,WAOI,YAAA,cAPJ,SAOI,YAAA,cAPJ,WAOI,YAAA,iBAPJ,MAOI,YAAA,YAPJ,OAOI,YAAA,eAPJ,SAOI,YAAA,cAPJ,OAOI,YAAA,YAPJ,YAOI,WAAA,eAPJ,UAOI,WAAA,gBAPJ,aAOI,WAAA,iBAPJ,sBAOI,gBAAA,eAPJ,2BAOI,gBAAA,oBAPJ,8BAOI,gBAAA,uBAPJ,gBAOI,eAAA,oBAPJ,gBAOI,eAAA,oBAPJ,iBAOI,eAAA,qBAPJ,WAOI,YAAA,iBAPJ,aAOI,YAAA,iBAPJ,YAOI,UAAA,qBAAA,WAAA,qBAPJ,cAIQ,kBAAA,EAGJ,MAAA,6DAPJ,gBAIQ,kBAAA,EAGJ,MAAA,+DAPJ,cAIQ,kBAAA,EAGJ,MAAA,6DAPJ,WAIQ,kBAAA,EAGJ,MAAA,0DAPJ,cAIQ,kBAAA,EAGJ,MAAA,6DAPJ,aAIQ,kBAAA,EAGJ,MAAA,4DAPJ,YAIQ,kBAAA,EAGJ,MAAA,2DAPJ,WAIQ,kBAAA,EAGJ,MAAA,0DAPJ,YAIQ,kBAAA,EAGJ,MAAA,2DAPJ,YAIQ,kBAAA,EAGJ,MAAA,2DAPJ,WAIQ,kBAAA,EAGJ,MAAA,0DAPJ,YAIQ,kBAAA,EAGJ,MAAA,kBAPJ,eAIQ,kBAAA,EAGJ,MAAA,yBAPJ,eAIQ,kBAAA,EAGJ,MAAA,+BAPJ,YAIQ,kBAAA,EAGJ,MAAA,kBAjBJ,iBACE,kBAAA,KADF,iBACE,kBAAA,IADF,iBACE,kBAAA,KADF,kBACE,kBAAA,EASF,YAIQ,gBAAA,EAGJ,iBAAA,2DAPJ,cAIQ,gBAAA,EAGJ,iBAAA,6DAPJ,YAIQ,gBAAA,EAGJ,iBAAA,2DAPJ,SAIQ,gBAAA,EAGJ,iBAAA,wDAPJ,YAIQ,gBAAA,EAGJ,iBAAA,2DAPJ,WAIQ,gBAAA,EAGJ,iBAAA,0DAPJ,UAIQ,gBAAA,EAGJ,iBAAA,yDAPJ,SAIQ,gBAAA,EAGJ,iBAAA,wDAPJ,UAIQ,gBAAA,EAGJ,iBAAA,yDAPJ,UAIQ,gBAAA,EAGJ,iBAAA,yDAPJ,SAIQ,gBAAA,EAGJ,iBAAA,wDAPJ,gBAIQ,gBAAA,EAGJ,iBAAA,sBAjBJ,eACE,gBAAA,IADF,eACE,gBAAA,KADF,eACE,gBAAA,IADF,eACE,gBAAA,KADF,gBACE,gBAAA,EASF,aAOI,iBAAA,6BAPJ,iBAOI,oBAAA,cAAA,iBAAA,cAAA,YAAA,cAPJ,kBAOI,oBAAA,eAAA,iBAAA,eAAA,YAAA,eAPJ,kBAOI,oBAAA,eAAA,iBAAA,eAAA,YAAA,eAPJ,SAOI,eAAA,eAPJ,SAOI,eAAA,eAPJ,SAOI,cAAA,iBAPJ,WAOI,cAAA,YAPJ,WAOI,cAAA,gBAPJ,WAOI,cAAA,iBAPJ,WAOI,cAAA,gBAPJ,gBAOI,cAAA,cAPJ,cAOI,cAAA,gBAPJ,aAOI,uBAAA,iBAAA,wBAAA,iBAPJ,aAOI,wBAAA,iBAAA,2BAAA,iBAPJ,gBAOI,2BAAA,iBAAA,0BAAA,iBAPJ,eAOI,0BAAA,iBAAA,uBAAA,iBAPJ,SAOI,WAAA,kBAPJ,WAOI,WAAA,iBzDPR,yByDAI,gBAOI,MAAA,eAPJ,cAOI,MAAA,gBAPJ,eAOI,MAAA,eAPJ,aAOI,QAAA,iBAPJ,mBAOI,QAAA,uBAPJ,YAOI,QAAA,gBAPJ,WAOI,QAAA,eAPJ,YAOI,QAAA,gBAPJ,gBAOI,QAAA,oBAPJ,iBAOI,QAAA,qBAPJ,WAOI,QAAA,eAPJ,kBAOI,QAAA,sBAPJ,WAOI,QAAA,eAPJ,cAOI,KAAA,EAAA,EAAA,eAPJ,aAOI,eAAA,cAPJ,gBAOI,eAAA,iBAPJ,qBAOI,eAAA,sBAPJ,wBAOI,eAAA,yBAPJ,gBAOI,UAAA,YAPJ,gBAOI,UAAA,YAPJ,kBAOI,YAAA,YAPJ,kBAOI,YAAA,YAPJ,cAOI,UAAA,eAPJ,gBAOI,UAAA,iBAPJ,sBAOI,UAAA,uBAPJ,UAOI,IAAA,YAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,gBAPJ,UAOI,IAAA,eAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,eAPJ,0BAOI,gBAAA,qBAPJ,wBAOI,gBAAA,mBAPJ,2BAOI,gBAAA,iBAPJ,4BAOI,gBAAA,wBAPJ,2BAOI,gBAAA,uBAPJ,2BAOI,gBAAA,uBAPJ,sBAOI,YAAA,qBAPJ,oBAOI,YAAA,mBAPJ,uBAOI,YAAA,iBAPJ,yBAOI,YAAA,mBAPJ,wBAOI,YAAA,kBAPJ,wBAOI,cAAA,qBAPJ,sBAOI,cAAA,mBAPJ,yBAOI,cAAA,iBAPJ,0BAOI,cAAA,wBAPJ,yBAOI,cAAA,uBAPJ,0BAOI,cAAA,kBAPJ,oBAOI,WAAA,eAPJ,qBAOI,WAAA,qBAPJ,mBAOI,WAAA,mBAPJ,sBAOI,WAAA,iBAPJ,wBAOI,WAAA,mBAPJ,uBAOI,WAAA,kBAPJ,gBAOI,MAAA,aAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,eAOI,MAAA,YAPJ,QAOI,OAAA,YAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,gBAPJ,QAOI,OAAA,eAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,eAPJ,WAOI,OAAA,eAPJ,SAOI,aAAA,YAAA,YAAA,YAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,gBAAA,YAAA,gBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,YAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,WAAA,YAAA,cAAA,YAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,gBAAA,cAAA,gBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,YAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,YAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,gBAPJ,SAOI,WAAA,eAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,eAPJ,YAOI,WAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,YAOI,aAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,YAOI,cAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,YAOI,YAAA,eAPJ,QAOI,QAAA,YAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,gBAPJ,QAOI,QAAA,eAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,eAPJ,SAOI,cAAA,YAAA,aAAA,YAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,gBAAA,aAAA,gBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,YAAA,YAAA,eAAA,YAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,gBAAA,eAAA,gBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,SAOI,eAAA,YAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,gBAPJ,SAOI,eAAA,eAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,eAOI,WAAA,eAPJ,aAOI,WAAA,gBAPJ,gBAOI,WAAA,kBzDPR,yByDAI,gBAOI,MAAA,eAPJ,cAOI,MAAA,gBAPJ,eAOI,MAAA,eAPJ,aAOI,QAAA,iBAPJ,mBAOI,QAAA,uBAPJ,YAOI,QAAA,gBAPJ,WAOI,QAAA,eAPJ,YAOI,QAAA,gBAPJ,gBAOI,QAAA,oBAPJ,iBAOI,QAAA,qBAPJ,WAOI,QAAA,eAPJ,kBAOI,QAAA,sBAPJ,WAOI,QAAA,eAPJ,cAOI,KAAA,EAAA,EAAA,eAPJ,aAOI,eAAA,cAPJ,gBAOI,eAAA,iBAPJ,qBAOI,eAAA,sBAPJ,wBAOI,eAAA,yBAPJ,gBAOI,UAAA,YAPJ,gBAOI,UAAA,YAPJ,kBAOI,YAAA,YAPJ,kBAOI,YAAA,YAPJ,cAOI,UAAA,eAPJ,gBAOI,UAAA,iBAPJ,sBAOI,UAAA,uBAPJ,UAOI,IAAA,YAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,gBAPJ,UAOI,IAAA,eAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,eAPJ,0BAOI,gBAAA,qBAPJ,wBAOI,gBAAA,mBAPJ,2BAOI,gBAAA,iBAPJ,4BAOI,gBAAA,wBAPJ,2BAOI,gBAAA,uBAPJ,2BAOI,gBAAA,uBAPJ,sBAOI,YAAA,qBAPJ,oBAOI,YAAA,mBAPJ,uBAOI,YAAA,iBAPJ,yBAOI,YAAA,mBAPJ,wBAOI,YAAA,kBAPJ,wBAOI,cAAA,qBAPJ,sBAOI,cAAA,mBAPJ,yBAOI,cAAA,iBAPJ,0BAOI,cAAA,wBAPJ,yBAOI,cAAA,uBAPJ,0BAOI,cAAA,kBAPJ,oBAOI,WAAA,eAPJ,qBAOI,WAAA,qBAPJ,mBAOI,WAAA,mBAPJ,sBAOI,WAAA,iBAPJ,wBAOI,WAAA,mBAPJ,uBAOI,WAAA,kBAPJ,gBAOI,MAAA,aAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,eAOI,MAAA,YAPJ,QAOI,OAAA,YAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,gBAPJ,QAOI,OAAA,eAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,eAPJ,WAOI,OAAA,eAPJ,SAOI,aAAA,YAAA,YAAA,YAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,gBAAA,YAAA,gBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,YAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,WAAA,YAAA,cAAA,YAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,gBAAA,cAAA,gBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,YAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,YAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,gBAPJ,SAOI,WAAA,eAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,eAPJ,YAOI,WAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,YAOI,aAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,YAOI,cAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,YAOI,YAAA,eAPJ,QAOI,QAAA,YAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,gBAPJ,QAOI,QAAA,eAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,eAPJ,SAOI,cAAA,YAAA,aAAA,YAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,gBAAA,aAAA,gBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,YAAA,YAAA,eAAA,YAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,gBAAA,eAAA,gBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,SAOI,eAAA,YAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,gBAPJ,SAOI,eAAA,eAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,eAOI,WAAA,eAPJ,aAOI,WAAA,gBAPJ,gBAOI,WAAA,kBzDPR,yByDAI,gBAOI,MAAA,eAPJ,cAOI,MAAA,gBAPJ,eAOI,MAAA,eAPJ,aAOI,QAAA,iBAPJ,mBAOI,QAAA,uBAPJ,YAOI,QAAA,gBAPJ,WAOI,QAAA,eAPJ,YAOI,QAAA,gBAPJ,gBAOI,QAAA,oBAPJ,iBAOI,QAAA,qBAPJ,WAOI,QAAA,eAPJ,kBAOI,QAAA,sBAPJ,WAOI,QAAA,eAPJ,cAOI,KAAA,EAAA,EAAA,eAPJ,aAOI,eAAA,cAPJ,gBAOI,eAAA,iBAPJ,qBAOI,eAAA,sBAPJ,wBAOI,eAAA,yBAPJ,gBAOI,UAAA,YAPJ,gBAOI,UAAA,YAPJ,kBAOI,YAAA,YAPJ,kBAOI,YAAA,YAPJ,cAOI,UAAA,eAPJ,gBAOI,UAAA,iBAPJ,sBAOI,UAAA,uBAPJ,UAOI,IAAA,YAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,gBAPJ,UAOI,IAAA,eAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,eAPJ,0BAOI,gBAAA,qBAPJ,wBAOI,gBAAA,mBAPJ,2BAOI,gBAAA,iBAPJ,4BAOI,gBAAA,wBAPJ,2BAOI,gBAAA,uBAPJ,2BAOI,gBAAA,uBAPJ,sBAOI,YAAA,qBAPJ,oBAOI,YAAA,mBAPJ,uBAOI,YAAA,iBAPJ,yBAOI,YAAA,mBAPJ,wBAOI,YAAA,kBAPJ,wBAOI,cAAA,qBAPJ,sBAOI,cAAA,mBAPJ,yBAOI,cAAA,iBAPJ,0BAOI,cAAA,wBAPJ,yBAOI,cAAA,uBAPJ,0BAOI,cAAA,kBAPJ,oBAOI,WAAA,eAPJ,qBAOI,WAAA,qBAPJ,mBAOI,WAAA,mBAPJ,sBAOI,WAAA,iBAPJ,wBAOI,WAAA,mBAPJ,uBAOI,WAAA,kBAPJ,gBAOI,MAAA,aAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,eAOI,MAAA,YAPJ,QAOI,OAAA,YAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,gBAPJ,QAOI,OAAA,eAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,eAPJ,WAOI,OAAA,eAPJ,SAOI,aAAA,YAAA,YAAA,YAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,gBAAA,YAAA,gBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,YAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,WAAA,YAAA,cAAA,YAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,gBAAA,cAAA,gBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,YAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,YAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,gBAPJ,SAOI,WAAA,eAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,eAPJ,YAOI,WAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,YAOI,aAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,YAOI,cAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,YAOI,YAAA,eAPJ,QAOI,QAAA,YAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,gBAPJ,QAOI,QAAA,eAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,eAPJ,SAOI,cAAA,YAAA,aAAA,YAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,gBAAA,aAAA,gBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,YAAA,YAAA,eAAA,YAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,gBAAA,eAAA,gBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,SAOI,eAAA,YAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,gBAPJ,SAOI,eAAA,eAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,eAOI,WAAA,eAPJ,aAOI,WAAA,gBAPJ,gBAOI,WAAA,kBzDPR,0ByDAI,gBAOI,MAAA,eAPJ,cAOI,MAAA,gBAPJ,eAOI,MAAA,eAPJ,aAOI,QAAA,iBAPJ,mBAOI,QAAA,uBAPJ,YAOI,QAAA,gBAPJ,WAOI,QAAA,eAPJ,YAOI,QAAA,gBAPJ,gBAOI,QAAA,oBAPJ,iBAOI,QAAA,qBAPJ,WAOI,QAAA,eAPJ,kBAOI,QAAA,sBAPJ,WAOI,QAAA,eAPJ,cAOI,KAAA,EAAA,EAAA,eAPJ,aAOI,eAAA,cAPJ,gBAOI,eAAA,iBAPJ,qBAOI,eAAA,sBAPJ,wBAOI,eAAA,yBAPJ,gBAOI,UAAA,YAPJ,gBAOI,UAAA,YAPJ,kBAOI,YAAA,YAPJ,kBAOI,YAAA,YAPJ,cAOI,UAAA,eAPJ,gBAOI,UAAA,iBAPJ,sBAOI,UAAA,uBAPJ,UAOI,IAAA,YAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,gBAPJ,UAOI,IAAA,eAPJ,UAOI,IAAA,iBAPJ,UAOI,IAAA,eAPJ,0BAOI,gBAAA,qBAPJ,wBAOI,gBAAA,mBAPJ,2BAOI,gBAAA,iBAPJ,4BAOI,gBAAA,wBAPJ,2BAOI,gBAAA,uBAPJ,2BAOI,gBAAA,uBAPJ,sBAOI,YAAA,qBAPJ,oBAOI,YAAA,mBAPJ,uBAOI,YAAA,iBAPJ,yBAOI,YAAA,mBAPJ,wBAOI,YAAA,kBAPJ,wBAOI,cAAA,qBAPJ,sBAOI,cAAA,mBAPJ,yBAOI,cAAA,iBAPJ,0BAOI,cAAA,wBAPJ,yBAOI,cAAA,uBAPJ,0BAOI,cAAA,kBAPJ,oBAOI,WAAA,eAPJ,qBAOI,WAAA,qBAPJ,mBAOI,WAAA,mBAPJ,sBAOI,WAAA,iBAPJ,wBAOI,WAAA,mBAPJ,uBAOI,WAAA,kBAPJ,gBAOI,MAAA,aAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,YAOI,MAAA,YAPJ,eAOI,MAAA,YAPJ,QAOI,OAAA,YAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,gBAPJ,QAOI,OAAA,eAPJ,QAOI,OAAA,iBAPJ,QAOI,OAAA,eAPJ,WAOI,OAAA,eAPJ,SAOI,aAAA,YAAA,YAAA,YAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,gBAAA,YAAA,gBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,aAAA,iBAAA,YAAA,iBAPJ,SAOI,aAAA,eAAA,YAAA,eAPJ,YAOI,aAAA,eAAA,YAAA,eAPJ,SAOI,WAAA,YAAA,cAAA,YAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,gBAAA,cAAA,gBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,iBAAA,cAAA,iBAPJ,SAOI,WAAA,eAAA,cAAA,eAPJ,YAOI,WAAA,eAAA,cAAA,eAPJ,SAOI,WAAA,YAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,gBAPJ,SAOI,WAAA,eAPJ,SAOI,WAAA,iBAPJ,SAOI,WAAA,eAPJ,YAOI,WAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,YAOI,aAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,YAOI,cAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,YAOI,YAAA,eAPJ,QAOI,QAAA,YAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,gBAPJ,QAOI,QAAA,eAPJ,QAOI,QAAA,iBAPJ,QAOI,QAAA,eAPJ,SAOI,cAAA,YAAA,aAAA,YAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,gBAAA,aAAA,gBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,cAAA,iBAAA,aAAA,iBAPJ,SAOI,cAAA,eAAA,aAAA,eAPJ,SAOI,YAAA,YAAA,eAAA,YAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,gBAAA,eAAA,gBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,iBAAA,eAAA,iBAPJ,SAOI,YAAA,eAAA,eAAA,eAPJ,SAOI,YAAA,YAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,gBAPJ,SAOI,YAAA,eAPJ,SAOI,YAAA,iBAPJ,SAOI,YAAA,eAPJ,SAOI,cAAA,YAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,gBAPJ,SAOI,cAAA,eAPJ,SAOI,cAAA,iBAPJ,SAOI,cAAA,eAPJ,SAOI,eAAA,YAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,gBAPJ,SAOI,eAAA,eAPJ,SAOI,eAAA,iBAPJ,SAOI,eAAA,eAPJ,SAOI,aAAA,YAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,gBAPJ,SAOI,aAAA,eAPJ,SAOI,aAAA,iBAPJ,SAOI,aAAA,eAPJ,eAOI,WAAA,eAPJ,aAOI,WAAA,gBAPJ,gBAOI,WAAA,kBzDPR,0ByDAI,iBAOI,MAAA,eAPJ,eAOI,MAAA,gBAPJ,gBAOI,MAAA,eAPJ,cAOI,QAAA,iBAPJ,oBAOI,QAAA,uBAPJ,aAOI,QAAA,gBAPJ,YAOI,QAAA,eAPJ,aAOI,QAAA,gBAPJ,iBAOI,QAAA,oBAPJ,kBAOI,QAAA,qBAPJ,YAOI,QAAA,eAPJ,mBAOI,QAAA,sBAPJ,YAOI,QAAA,eAPJ,eAOI,KAAA,EAAA,EAAA,eAPJ,cAOI,eAAA,cAPJ,iBAOI,eAAA,iBAPJ,sBAOI,eAAA,sBAPJ,yBAOI,eAAA,yBAPJ,iBAOI,UAAA,YAPJ,iBAOI,UAAA,YAPJ,mBAOI,YAAA,YAPJ,mBAOI,YAAA,YAPJ,eAOI,UAAA,eAPJ,iBAOI,UAAA,iBAPJ,uBAOI,UAAA,uBAPJ,WAOI,IAAA,YAPJ,WAOI,IAAA,iBAPJ,WAOI,IAAA,gBAPJ,WAOI,IAAA,eAPJ,WAOI,IAAA,iBAPJ,WAOI,IAAA,eAPJ,2BAOI,gBAAA,qBAPJ,yBAOI,gBAAA,mBAPJ,4BAOI,gBAAA,iBAPJ,6BAOI,gBAAA,wBAPJ,4BAOI,gBAAA,uBAPJ,4BAOI,gBAAA,uBAPJ,uBAOI,YAAA,qBAPJ,qBAOI,YAAA,mBAPJ,wBAOI,YAAA,iBAPJ,0BAOI,YAAA,mBAPJ,yBAOI,YAAA,kBAPJ,yBAOI,cAAA,qBAPJ,uBAOI,cAAA,mBAPJ,0BAOI,cAAA,iBAPJ,2BAOI,cAAA,wBAPJ,0BAOI,cAAA,uBAPJ,2BAOI,cAAA,kBAPJ,qBAOI,WAAA,eAPJ,sBAOI,WAAA,qBAPJ,oBAOI,WAAA,mBAPJ,uBAOI,WAAA,iBAPJ,yBAOI,WAAA,mBAPJ,wBAOI,WAAA,kBAPJ,iBAOI,MAAA,aAPJ,aAOI,MAAA,YAPJ,aAOI,MAAA,YAPJ,aAOI,MAAA,YAPJ,aAOI,MAAA,YAPJ,aAOI,MAAA,YAPJ,aAOI,MAAA,YAPJ,gBAOI,MAAA,YAPJ,SAOI,OAAA,YAPJ,SAOI,OAAA,iBAPJ,SAOI,OAAA,gBAPJ,SAOI,OAAA,eAPJ,SAOI,OAAA,iBAPJ,SAOI,OAAA,eAPJ,YAOI,OAAA,eAPJ,UAOI,aAAA,YAAA,YAAA,YAPJ,UAOI,aAAA,iBAAA,YAAA,iBAPJ,UAOI,aAAA,gBAAA,YAAA,gBAPJ,UAOI,aAAA,eAAA,YAAA,eAPJ,UAOI,aAAA,iBAAA,YAAA,iBAPJ,UAOI,aAAA,eAAA,YAAA,eAPJ,aAOI,aAAA,eAAA,YAAA,eAPJ,UAOI,WAAA,YAAA,cAAA,YAPJ,UAOI,WAAA,iBAAA,cAAA,iBAPJ,UAOI,WAAA,gBAAA,cAAA,gBAPJ,UAOI,WAAA,eAAA,cAAA,eAPJ,UAOI,WAAA,iBAAA,cAAA,iBAPJ,UAOI,WAAA,eAAA,cAAA,eAPJ,aAOI,WAAA,eAAA,cAAA,eAPJ,UAOI,WAAA,YAPJ,UAOI,WAAA,iBAPJ,UAOI,WAAA,gBAPJ,UAOI,WAAA,eAPJ,UAOI,WAAA,iBAPJ,UAOI,WAAA,eAPJ,aAOI,WAAA,eAPJ,UAOI,aAAA,YAPJ,UAOI,aAAA,iBAPJ,UAOI,aAAA,gBAPJ,UAOI,aAAA,eAPJ,UAOI,aAAA,iBAPJ,UAOI,aAAA,eAPJ,aAOI,aAAA,eAPJ,UAOI,cAAA,YAPJ,UAOI,cAAA,iBAPJ,UAOI,cAAA,gBAPJ,UAOI,cAAA,eAPJ,UAOI,cAAA,iBAPJ,UAOI,cAAA,eAPJ,aAOI,cAAA,eAPJ,UAOI,YAAA,YAPJ,UAOI,YAAA,iBAPJ,UAOI,YAAA,gBAPJ,UAOI,YAAA,eAPJ,UAOI,YAAA,iBAPJ,UAOI,YAAA,eAPJ,aAOI,YAAA,eAPJ,SAOI,QAAA,YAPJ,SAOI,QAAA,iBAPJ,SAOI,QAAA,gBAPJ,SAOI,QAAA,eAPJ,SAOI,QAAA,iBAPJ,SAOI,QAAA,eAPJ,UAOI,cAAA,YAAA,aAAA,YAPJ,UAOI,cAAA,iBAAA,aAAA,iBAPJ,UAOI,cAAA,gBAAA,aAAA,gBAPJ,UAOI,cAAA,eAAA,aAAA,eAPJ,UAOI,cAAA,iBAAA,aAAA,iBAPJ,UAOI,cAAA,eAAA,aAAA,eAPJ,UAOI,YAAA,YAAA,eAAA,YAPJ,UAOI,YAAA,iBAAA,eAAA,iBAPJ,UAOI,YAAA,gBAAA,eAAA,gBAPJ,UAOI,YAAA,eAAA,eAAA,eAPJ,UAOI,YAAA,iBAAA,eAAA,iBAPJ,UAOI,YAAA,eAAA,eAAA,eAPJ,UAOI,YAAA,YAPJ,UAOI,YAAA,iBAPJ,UAOI,YAAA,gBAPJ,UAOI,YAAA,eAPJ,UAOI,YAAA,iBAPJ,UAOI,YAAA,eAPJ,UAOI,cAAA,YAPJ,UAOI,cAAA,iBAPJ,UAOI,cAAA,gBAPJ,UAOI,cAAA,eAPJ,UAOI,cAAA,iBAPJ,UAOI,cAAA,eAPJ,UAOI,eAAA,YAPJ,UAOI,eAAA,iBAPJ,UAOI,eAAA,gBAPJ,UAOI,eAAA,eAPJ,UAOI,eAAA,iBAPJ,UAOI,eAAA,eAPJ,UAOI,aAAA,YAPJ,UAOI,aAAA,iBAPJ,UAOI,aAAA,gBAPJ,UAOI,aAAA,eAPJ,UAOI,aAAA,iBAPJ,UAOI,aAAA,eAPJ,gBAOI,WAAA,eAPJ,cAOI,WAAA,gBAPJ,iBAOI,WAAA,kBCnDZ,0BD4CQ,MAOI,UAAA,iBAPJ,MAOI,UAAA,eAPJ,MAOI,UAAA,kBAPJ,MAOI,UAAA,kBChCZ,aDyBQ,gBAOI,QAAA,iBAPJ,sBAOI,QAAA,uBAPJ,eAOI,QAAA,gBAPJ,cAOI,QAAA,eAPJ,eAOI,QAAA,gBAPJ,mBAOI,QAAA,oBAPJ,oBAOI,QAAA,qBAPJ,cAOI,QAAA,eAPJ,qBAOI,QAAA,sBAPJ,cAOI,QAAA","sourcesContent":["/*!\n * Bootstrap v5.1.0 (https://getbootstrap.com/)\n * Copyright 2011-2021 The Bootstrap Authors\n * Copyright 2011-2021 Twitter, Inc.\n * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)\n */\n\n// scss-docs-start import-stack\n// Configuration\n@import \"functions\";\n@import \"variables\";\n@import \"mixins\";\n@import \"utilities\";\n\n// Layout & components\n@import \"root\";\n@import \"reboot\";\n@import \"type\";\n@import \"images\";\n@import \"containers\";\n@import \"grid\";\n@import \"tables\";\n@import \"forms\";\n@import \"buttons\";\n@import \"transitions\";\n@import \"dropdown\";\n@import \"button-group\";\n@import \"nav\";\n@import \"navbar\";\n@import \"card\";\n@import \"accordion\";\n@import \"breadcrumb\";\n@import \"pagination\";\n@import \"badge\";\n@import \"alert\";\n@import \"progress\";\n@import \"list-group\";\n@import \"close\";\n@import \"toasts\";\n@import \"modal\";\n@import \"tooltip\";\n@import \"popover\";\n@import \"carousel\";\n@import \"spinners\";\n@import \"offcanvas\";\n@import \"placeholders\";\n\n// Helpers\n@import \"helpers\";\n\n// Utilities\n@import \"utilities/api\";\n// scss-docs-end import-stack\n",":root {\n // Note: Custom variable values only support SassScript inside `#{}`.\n\n // Colors\n //\n // Generate palettes for full colors, grays, and theme colors.\n\n @each $color, $value in $colors {\n --#{$variable-prefix}#{$color}: #{$value};\n }\n\n @each $color, $value in $grays {\n --#{$variable-prefix}gray-#{$color}: #{$value};\n }\n\n @each $color, $value in $theme-colors {\n --#{$variable-prefix}#{$color}: #{$value};\n }\n\n @each $color, $value in $theme-colors-rgb {\n --#{$variable-prefix}#{$color}-rgb: #{$value};\n }\n\n --#{$variable-prefix}white-rgb: #{to-rgb($white)};\n --#{$variable-prefix}black-rgb: #{to-rgb($black)};\n --#{$variable-prefix}body-rgb: #{to-rgb($body-color)};\n\n // Fonts\n\n // Note: Use `inspect` for lists so that quoted items keep the quotes.\n // See https://github.com/sass/sass/issues/2383#issuecomment-336349172\n --#{$variable-prefix}font-sans-serif: #{inspect($font-family-sans-serif)};\n --#{$variable-prefix}font-monospace: #{inspect($font-family-monospace)};\n --#{$variable-prefix}gradient: #{$gradient};\n\n // Root and body\n // stylelint-disable custom-property-empty-line-before\n // scss-docs-start root-body-variables\n @if $font-size-root != null {\n --#{$variable-prefix}root-font-size: #{$font-size-root};\n }\n --#{$variable-prefix}body-font-family: #{$font-family-base};\n --#{$variable-prefix}body-font-size: #{$font-size-base};\n --#{$variable-prefix}body-font-weight: #{$font-weight-base};\n --#{$variable-prefix}body-line-height: #{$line-height-base};\n --#{$variable-prefix}body-color: #{$body-color};\n @if $body-text-align != null {\n --#{$variable-prefix}body-text-align: #{$body-text-align};\n }\n --#{$variable-prefix}body-bg: #{$body-bg};\n // scss-docs-end root-body-variables\n // stylelint-enable custom-property-empty-line-before\n}\n","// stylelint-disable declaration-no-important, selector-no-qualifying-type, property-no-vendor-prefix\n\n\n// Reboot\n//\n// Normalization of HTML elements, manually forked from Normalize.css to remove\n// styles targeting irrelevant browsers while applying new styles.\n//\n// Normalize is licensed MIT. https://github.com/necolas/normalize.css\n\n\n// Document\n//\n// Change from `box-sizing: content-box` so that `width` is not affected by `padding` or `border`.\n\n*,\n*::before,\n*::after {\n box-sizing: border-box;\n}\n\n\n// Root\n//\n// Ability to the value of the root font sizes, affecting the value of `rem`.\n// null by default, thus nothing is generated.\n\n:root {\n @if $font-size-root != null {\n font-size: var(--#{$variable-prefix}-root-font-size);\n }\n\n @if $enable-smooth-scroll {\n @media (prefers-reduced-motion: no-preference) {\n scroll-behavior: smooth;\n }\n }\n}\n\n\n// Body\n//\n// 1. Remove the margin in all browsers.\n// 2. As a best practice, apply a default `background-color`.\n// 3. Prevent adjustments of font size after orientation changes in iOS.\n// 4. Change the default tap highlight to be completely transparent in iOS.\n\n// scss-docs-start reboot-body-rules\nbody {\n margin: 0; // 1\n font-family: var(--#{$variable-prefix}body-font-family);\n @include font-size(var(--#{$variable-prefix}body-font-size));\n font-weight: var(--#{$variable-prefix}body-font-weight);\n line-height: var(--#{$variable-prefix}body-line-height);\n color: var(--#{$variable-prefix}body-color);\n text-align: var(--#{$variable-prefix}body-text-align);\n background-color: var(--#{$variable-prefix}body-bg); // 2\n -webkit-text-size-adjust: 100%; // 3\n -webkit-tap-highlight-color: rgba($black, 0); // 4\n}\n// scss-docs-end reboot-body-rules\n\n\n// Content grouping\n//\n// 1. Reset Firefox's gray color\n// 2. Set correct height and prevent the `size` attribute to make the `hr` look like an input field\n\nhr {\n margin: $hr-margin-y 0;\n color: $hr-color; // 1\n background-color: currentColor;\n border: 0;\n opacity: $hr-opacity;\n}\n\nhr:not([size]) {\n height: $hr-height; // 2\n}\n\n\n// Typography\n//\n// 1. Remove top margins from headings\n// By default, `

`-`

` all receive top and bottom margins. We nuke the top\n// margin for easier control within type scales as it avoids margin collapsing.\n\n%heading {\n margin-top: 0; // 1\n margin-bottom: $headings-margin-bottom;\n font-family: $headings-font-family;\n font-style: $headings-font-style;\n font-weight: $headings-font-weight;\n line-height: $headings-line-height;\n color: $headings-color;\n}\n\nh1 {\n @extend %heading;\n @include font-size($h1-font-size);\n}\n\nh2 {\n @extend %heading;\n @include font-size($h2-font-size);\n}\n\nh3 {\n @extend %heading;\n @include font-size($h3-font-size);\n}\n\nh4 {\n @extend %heading;\n @include font-size($h4-font-size);\n}\n\nh5 {\n @extend %heading;\n @include font-size($h5-font-size);\n}\n\nh6 {\n @extend %heading;\n @include font-size($h6-font-size);\n}\n\n\n// Reset margins on paragraphs\n//\n// Similarly, the top margin on `

`s get reset. However, we also reset the\n// bottom margin to use `rem` units instead of `em`.\n\np {\n margin-top: 0;\n margin-bottom: $paragraph-margin-bottom;\n}\n\n\n// Abbreviations\n//\n// 1. Duplicate behavior to the data-bs-* attribute for our tooltip plugin\n// 2. Add the correct text decoration in Chrome, Edge, Opera, and Safari.\n// 3. Add explicit cursor to indicate changed behavior.\n// 4. Prevent the text-decoration to be skipped.\n\nabbr[title],\nabbr[data-bs-original-title] { // 1\n text-decoration: underline dotted; // 2\n cursor: help; // 3\n text-decoration-skip-ink: none; // 4\n}\n\n\n// Address\n\naddress {\n margin-bottom: 1rem;\n font-style: normal;\n line-height: inherit;\n}\n\n\n// Lists\n\nol,\nul {\n padding-left: 2rem;\n}\n\nol,\nul,\ndl {\n margin-top: 0;\n margin-bottom: 1rem;\n}\n\nol ol,\nul ul,\nol ul,\nul ol {\n margin-bottom: 0;\n}\n\ndt {\n font-weight: $dt-font-weight;\n}\n\n// 1. Undo browser default\n\ndd {\n margin-bottom: .5rem;\n margin-left: 0; // 1\n}\n\n\n// Blockquote\n\nblockquote {\n margin: 0 0 1rem;\n}\n\n\n// Strong\n//\n// Add the correct font weight in Chrome, Edge, and Safari\n\nb,\nstrong {\n font-weight: $font-weight-bolder;\n}\n\n\n// Small\n//\n// Add the correct font size in all browsers\n\nsmall {\n @include font-size($small-font-size);\n}\n\n\n// Mark\n\nmark {\n padding: $mark-padding;\n background-color: $mark-bg;\n}\n\n\n// Sub and Sup\n//\n// Prevent `sub` and `sup` elements from affecting the line height in\n// all browsers.\n\nsub,\nsup {\n position: relative;\n @include font-size($sub-sup-font-size);\n line-height: 0;\n vertical-align: baseline;\n}\n\nsub { bottom: -.25em; }\nsup { top: -.5em; }\n\n\n// Links\n\na {\n color: $link-color;\n text-decoration: $link-decoration;\n\n &:hover {\n color: $link-hover-color;\n text-decoration: $link-hover-decoration;\n }\n}\n\n// And undo these styles for placeholder links/named anchors (without href).\n// It would be more straightforward to just use a[href] in previous block, but that\n// causes specificity issues in many other styles that are too complex to fix.\n// See https://github.com/twbs/bootstrap/issues/19402\n\na:not([href]):not([class]) {\n &,\n &:hover {\n color: inherit;\n text-decoration: none;\n }\n}\n\n\n// Code\n\npre,\ncode,\nkbd,\nsamp {\n font-family: $font-family-code;\n @include font-size(1em); // Correct the odd `em` font sizing in all browsers.\n direction: ltr #{\"/* rtl:ignore */\"};\n unicode-bidi: bidi-override;\n}\n\n// 1. Remove browser default top margin\n// 2. Reset browser default of `1em` to use `rem`s\n// 3. Don't allow content to break outside\n\npre {\n display: block;\n margin-top: 0; // 1\n margin-bottom: 1rem; // 2\n overflow: auto; // 3\n @include font-size($code-font-size);\n color: $pre-color;\n\n // Account for some code outputs that place code tags in pre tags\n code {\n @include font-size(inherit);\n color: inherit;\n word-break: normal;\n }\n}\n\ncode {\n @include font-size($code-font-size);\n color: $code-color;\n word-wrap: break-word;\n\n // Streamline the style when inside anchors to avoid broken underline and more\n a > & {\n color: inherit;\n }\n}\n\nkbd {\n padding: $kbd-padding-y $kbd-padding-x;\n @include font-size($kbd-font-size);\n color: $kbd-color;\n background-color: $kbd-bg;\n @include border-radius($border-radius-sm);\n\n kbd {\n padding: 0;\n @include font-size(1em);\n font-weight: $nested-kbd-font-weight;\n }\n}\n\n\n// Figures\n//\n// Apply a consistent margin strategy (matches our type styles).\n\nfigure {\n margin: 0 0 1rem;\n}\n\n\n// Images and content\n\nimg,\nsvg {\n vertical-align: middle;\n}\n\n\n// Tables\n//\n// Prevent double borders\n\ntable {\n caption-side: bottom;\n border-collapse: collapse;\n}\n\ncaption {\n padding-top: $table-cell-padding-y;\n padding-bottom: $table-cell-padding-y;\n color: $table-caption-color;\n text-align: left;\n}\n\n// 1. Removes font-weight bold by inheriting\n// 2. Matches default `` alignment by inheriting `text-align`.\n// 3. Fix alignment for Safari\n\nth {\n font-weight: $table-th-font-weight; // 1\n text-align: inherit; // 2\n text-align: -webkit-match-parent; // 3\n}\n\nthead,\ntbody,\ntfoot,\ntr,\ntd,\nth {\n border-color: inherit;\n border-style: solid;\n border-width: 0;\n}\n\n\n// Forms\n//\n// 1. Allow labels to use `margin` for spacing.\n\nlabel {\n display: inline-block; // 1\n}\n\n// Remove the default `border-radius` that macOS Chrome adds.\n// See https://github.com/twbs/bootstrap/issues/24093\n\nbutton {\n // stylelint-disable-next-line property-disallowed-list\n border-radius: 0;\n}\n\n// Explicitly remove focus outline in Chromium when it shouldn't be\n// visible (e.g. as result of mouse click or touch tap). It already\n// should be doing this automatically, but seems to currently be\n// confused and applies its very visible two-tone outline anyway.\n\nbutton:focus:not(:focus-visible) {\n outline: 0;\n}\n\n// 1. Remove the margin in Firefox and Safari\n\ninput,\nbutton,\nselect,\noptgroup,\ntextarea {\n margin: 0; // 1\n font-family: inherit;\n @include font-size(inherit);\n line-height: inherit;\n}\n\n// Remove the inheritance of text transform in Firefox\nbutton,\nselect {\n text-transform: none;\n}\n// Set the cursor for non-` + + + + diff --git a/playground/DashboardToy/DashboradToy.AppHost/DashboardToy.AppHost.csproj b/playground/DashboardToy/DashboradToy.AppHost/DashboardToy.AppHost.csproj new file mode 100644 index 0000000000..4fa3be891b --- /dev/null +++ b/playground/DashboardToy/DashboradToy.AppHost/DashboardToy.AppHost.csproj @@ -0,0 +1,22 @@ + + + + Exe + net8.0 + enable + enable + true + 6a521b87-2bf9-4af8-b7c7-4947536e1d50 + + + + + + + + + + + + + diff --git a/playground/DashboardToy/DashboradToy.AppHost/Program.cs b/playground/DashboardToy/DashboradToy.AppHost/Program.cs new file mode 100644 index 0000000000..b322f314bd --- /dev/null +++ b/playground/DashboardToy/DashboradToy.AppHost/Program.cs @@ -0,0 +1,13 @@ +using Projects; + +var builder = DistributedApplication.CreateBuilder(args); +var redis = builder.AddRedis("orleans-redis"); + +var orleans = builder.AddOrleans("cluster") + .WithClustering(redis); + +builder.AddProject("frontend") + .WithReference(orleans) + .WithReplicas(5); + +builder.Build().Run(); diff --git a/playground/DashboardToy/DashboradToy.AppHost/Properties/launchSettings.json b/playground/DashboardToy/DashboradToy.AppHost/Properties/launchSettings.json new file mode 100644 index 0000000000..f6511e66f9 --- /dev/null +++ b/playground/DashboardToy/DashboradToy.AppHost/Properties/launchSettings.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "https": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17234;http://localhost:15087", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21284", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22143" + } + }, + "http": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "http://localhost:15087", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "http://localhost:19030", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "http://localhost:20232" + } + } + } +} diff --git a/playground/DashboardToy/DashboradToy.AppHost/appsettings.Development.json b/playground/DashboardToy/DashboradToy.AppHost/appsettings.Development.json new file mode 100644 index 0000000000..0c208ae918 --- /dev/null +++ b/playground/DashboardToy/DashboradToy.AppHost/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + } +} diff --git a/playground/DashboardToy/DashboradToy.AppHost/appsettings.json b/playground/DashboardToy/DashboradToy.AppHost/appsettings.json new file mode 100644 index 0000000000..31c092aa45 --- /dev/null +++ b/playground/DashboardToy/DashboradToy.AppHost/appsettings.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning", + "Aspire.Hosting.Dcp": "Warning" + } + } +} diff --git a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs index a977fef48d..df91af774f 100644 --- a/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs +++ b/src/Orleans.Core/Placement/Rebalancing/IActivationRebalancerSystemTarget.cs @@ -33,6 +33,11 @@ static IActivationRebalancerSystemTarget GetReference(IGrainFactory grainFactory /// For use in testing only! ///

ValueTask SetActivationCountOffset(int activationCountOffset); + + /// + /// For diagnostics only. + /// + ValueTask> GetGrainCallFrequencies(); } // We use a readonly struct so that we can fully decouple the message-passing and potentially modifications to the Silo fields. diff --git a/src/Orleans.Core/SystemTargetInterfaces/IManagementGrain.cs b/src/Orleans.Core/SystemTargetInterfaces/IManagementGrain.cs index 2c77d8a851..8d99ab2556 100644 --- a/src/Orleans.Core/SystemTargetInterfaces/IManagementGrain.cs +++ b/src/Orleans.Core/SystemTargetInterfaces/IManagementGrain.cs @@ -124,5 +124,25 @@ public interface IManagementGrain : IGrainWithIntegerKey, IVersionManager /// The type. /// A list of all active grains of the specified type. ValueTask> GetActiveGrains(GrainType type); + + Task> GetGrainCallFrequencies(SiloAddress[] hostsIds = null); + ValueTask ResetGrainCallFrequencies(SiloAddress[] hostsIds = null); + } + + [GenerateSerializer] + [Alias("Orleans.Runtime.GrainCallFrequency")] + [Immutable] + public struct GrainCallFrequency + { + [Id(0)] + public GrainId SourceGrain { get; set; } + [Id(1)] + public GrainId TargetGrain { get; set; } + [Id(2)] + public SiloAddress SourceHost { get; set; } + [Id(3)] + public SiloAddress TargetHost { get; set; } + [Id(4)] + public ulong CallCount { get; set; } } } diff --git a/src/Orleans.Runtime/Catalog/ActivationData.cs b/src/Orleans.Runtime/Catalog/ActivationData.cs index 89ceb741cc..f5712651e9 100644 --- a/src/Orleans.Runtime/Catalog/ActivationData.cs +++ b/src/Orleans.Runtime/Catalog/ActivationData.cs @@ -870,7 +870,7 @@ void ProcessPendingRequests() // If the activation is not valid, reject all pending messages except for local-only messages. // Local-only messages are used for internal system operations and should not be rejected. - if (State != ActivationState.Valid && !message.IsLocalOnly) + if (State != ActivationState.Valid && !(message.IsLocalOnly && State != ActivationState.Invalid)) { ProcessRequestsToInvalidActivation(); break; diff --git a/src/Orleans.Runtime/Catalog/ActivationDirectory.cs b/src/Orleans.Runtime/Catalog/ActivationDirectory.cs index 7e43f51453..0d3a9d7fd6 100644 --- a/src/Orleans.Runtime/Catalog/ActivationDirectory.cs +++ b/src/Orleans.Runtime/Catalog/ActivationDirectory.cs @@ -1,3 +1,4 @@ +#nullable enable using System; using System.Collections; using System.Collections.Concurrent; @@ -20,7 +21,7 @@ public ActivationDirectory() public int Count => _activationsCount; - public IGrainContext FindTarget(GrainId key) + public IGrainContext? FindTarget(GrainId key) { _activations.TryGetValue(key, out var result); return result; diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index 533b52054f..4ee5a3ccbc 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -36,7 +36,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public static readonly TimeSpan DEFAULT_MINUMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(0.5); + public static readonly TimeSpan DEFAULT_MINUMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(1); /// /// The maximum time between initiating a rebalancing cycle. @@ -50,7 +50,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public static readonly TimeSpan DEFAULT_MAXIMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(1); + public static readonly TimeSpan DEFAULT_MAXIMUM_REBALANCING_PERIOD = TimeSpan.FromMinutes(2); /// /// The minimum time needed for a silo to recover from a previous rebalancing. @@ -61,7 +61,7 @@ public sealed class ActiveRebalancingOptions /// /// The default value of . /// - public static readonly TimeSpan DEFAULT_RECOVERY_PERIOD = TimeSpan.FromMinutes(0.5); + public static readonly TimeSpan DEFAULT_RECOVERY_PERIOD = TimeSpan.FromMinutes(1); /// /// The maximum number of unprocessed edges to buffer. If this number is exceeded, the oldest edges will be discarded. diff --git a/src/Orleans.Runtime/Configuration/Options/ResourceOptimizedPlacementOptions.cs b/src/Orleans.Runtime/Configuration/Options/ResourceOptimizedPlacementOptions.cs index 27b60f7b29..46418f8f9b 100644 --- a/src/Orleans.Runtime/Configuration/Options/ResourceOptimizedPlacementOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ResourceOptimizedPlacementOptions.cs @@ -65,7 +65,7 @@ public sealed class ResourceOptimizedPlacementOptions /// /// The default value of . /// - public const int DEFAULT_MAX_AVAILABLE_MEMORY_WEIGHT = 10; + public const int DEFAULT_MAX_AVAILABLE_MEMORY_WEIGHT = 0; /// /// The specified margin for which: if two silos (one of them being the local to the current pending activation), have a utilization score that should be considered "the same" within this margin. diff --git a/src/Orleans.Runtime/Core/ManagementGrain.cs b/src/Orleans.Runtime/Core/ManagementGrain.cs index 0e8767b97f..a78657d4e8 100644 --- a/src/Orleans.Runtime/Core/ManagementGrain.cs +++ b/src/Orleans.Runtime/Core/ManagementGrain.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using Orleans.Concurrency; using Orleans.Metadata; +using Orleans.Placement.Rebalancing; using Orleans.Providers; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.MembershipService; @@ -371,5 +372,49 @@ public async ValueTask> GetActiveGrains(GrainType grainType) return results; } + + public async Task> GetGrainCallFrequencies(SiloAddress[] hostsIds = null) + { + if (hostsIds == null) + { + var hosts = await GetHosts(true); + hostsIds = [.. hosts.Keys]; + } + + var results = new List(); + foreach (var host in hostsIds) + { + var siloBalancer = IActivationRebalancerSystemTarget.GetReference(internalGrainFactory, host); + var frequencies = await siloBalancer.GetGrainCallFrequencies(); + foreach (var frequency in frequencies) + { + results.Add(new GrainCallFrequency + { + SourceGrain = frequency.Item1.Source.Id, + TargetGrain = frequency.Item1.Target.Id, + SourceHost = frequency.Item1.Source.Silo, + TargetHost = frequency.Item1.Target.Silo, + CallCount = frequency.Item2 + }); + } + } + + return results; + } + + public async ValueTask ResetGrainCallFrequencies(SiloAddress[] hostsIds = null) + { + if (hostsIds == null) + { + var hosts = await GetHosts(true); + hostsIds = [.. hosts.Keys]; + } + + foreach (var host in hostsIds) + { + var siloBalancer = IActivationRebalancerSystemTarget.GetReference(internalGrainFactory, host); + await siloBalancer.ResetCounters(); + } + } } } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index ef8744327a..0ca8d9e2b5 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -65,7 +65,7 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) } EdgeVertex sourceVertex; - if (_anchoredGrainIds.Contains(message.SendingGrain)) + if (_anchoredGrainIds.Contains(message.SendingGrain) && Silo.Equals(message.SendingSilo)) { sourceVertex = new(GrainId, Silo, isMigratable: false); } @@ -75,7 +75,7 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) } EdgeVertex destinationVertex; - if (_anchoredGrainIds.Contains(message.TargetGrain)) + if (_anchoredGrainIds.Contains(message.TargetGrain) && Silo.Equals(message.TargetSilo)) { destinationVertex = new(GrainId, Silo, isMigratable: false); } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index a163f69ecf..199e4f872c 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -29,6 +29,7 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe private readonly IInternalGrainFactory _grainFactory; private readonly IRebalancingMessageFilter _messageFilter; private readonly IImbalanceToleranceRule _toleranceRule; + private readonly IActivationMigrationManager _migrationManager; private readonly ActivationDirectory _activationDirectory; private readonly TimeProvider _timeProvider; private readonly ActiveRebalancingOptions _options; @@ -48,6 +49,7 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe IInternalGrainFactory internalGrainFactory, IRebalancingMessageFilter messageFilter, IImbalanceToleranceRule toleranceRule, + IActivationMigrationManager migrationManager, ActivationDirectory activationDirectory, Catalog catalog, IOptions options, @@ -60,6 +62,7 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe _grainFactory = internalGrainFactory; _messageFilter = messageFilter; _toleranceRule = toleranceRule; + _migrationManager = migrationManager; _activationDirectory = activationDirectory; _timeProvider = timeProvider; _edgeWeights = new((int)options.Value.MaxEdgeCount); @@ -87,6 +90,7 @@ public ValueTask ResetCounters() { _pendingMessages.Clear(); _edgeWeights.Clear(); + _anchoredGrainIds.Reset(); return ValueTask.CompletedTask; } @@ -301,6 +305,7 @@ bool TryMigrateLocalToRemote() // We will perform any future operations assuming the vector is remote. chosenVertex.Location = VertexLocation.Remote; + Debug.Assert(((IHeapElement)chosenVertex).HeapIndex == -1); return true; } @@ -339,7 +344,7 @@ bool TryMigrateRemoteToLocal() bool TryMigrateCore(MaxHeap sourceHeap, int localDelta, int remoteDelta, [NotNullWhen(true)] out CandidateVertexHeapElement? chosenVertex) { var anticipatedImbalance = CalculateImbalance(localActivations + localDelta, remoteActivations + remoteDelta); - if (anticipatedImbalance >= initialImbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) + if (anticipatedImbalance >= imbalance && !_toleranceRule.IsSatisfiedBy((uint)anticipatedImbalance)) { // Taking from this heap would not improve imbalance. chosenVertex = null; @@ -376,7 +381,7 @@ bool TryMigrateCore(MaxHeap sourceHeap, int localDel } } - private static int CalculateImbalance(int left, int right) => (int)Math.Abs(Math.Abs((long)left) - Math.Abs((long)right)); + private static int CalculateImbalance(int left, int right) => Math.Abs(Math.Abs(left) - Math.Abs(right)); private static (MaxHeap Local, MaxHeap Remote) CreateCandidateHeaps(List local, ImmutableArray remote) { Dictionary sourceIndex = new(local.Count + remote.Length); @@ -454,31 +459,29 @@ static CandidateVertexHeapElement GetOrAddVertex(Dictionary giving, ImmutableArray accepting, SiloAddress targetSilo, HashSet newlyAnchoredGrains) { // The protocol concluded that 'this' silo should take on 'set', so we hint to the director accordingly. - RequestContext.Set(IPlacementDirector.PlacementHintKey, targetSilo); - List migrationTasks = []; - - foreach (var grainId in giving) - { - migrationTasks.Add(_grainFactory.GetGrain(grainId).Cast().MigrateOnIdle().AsTask()); - } - try { - await Task.WhenAll(migrationTasks); + Dictionary migrationRequestContext = new() { [IPlacementDirector.PlacementHintKey] = targetSilo }; + foreach (var grainId in giving) + { + if (_activationDirectory.FindTarget(grainId) is { } localActivation) + { + localActivation.Migrate(migrationRequestContext); + } + } } - catch + catch (Exception exception) { // This should happen rarely, but at this point we cant really do much, as its out of our control. // Even if some fail, the algorithm will eventually run again, so activations will have more chances to migrate. - var aggEx = new AggregateException(migrationTasks.Select(t => t.Exception).Where(ex => ex is not null)!); - LogErrorOnMigratingActivations(aggEx); + LogErrorOnMigratingActivations(exception); } // Avoid mutating the source while enumerating it. var iterations = 0; var toRemove = new List(); - var affected = new HashSet(giving.Length + accepting.Length); + _logger.LogInformation("Adding {NewlyAnchoredGrains} newly anchored grains to set on host {Silo}. EdgeWeights contains {EdgeWeightCount} elements.", newlyAnchoredGrains.Count, Silo, _edgeWeights.Count); foreach (var id in newlyAnchoredGrains) { @@ -498,7 +501,7 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr var yieldStopwatch = CoarseStopwatch.StartNew(); if (affected.Count > 0) { - foreach (var (edge, count, error) in _edgeWeights.Elements) + foreach (var (edge, _, _) in _edgeWeights.Elements) { if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id) || _anchoredGrainIds.Contains(edge.Source.Id) || _anchoredGrainIds.Contains(edge.Target.Id)) { @@ -579,13 +582,13 @@ private List GetCandidatesForSilo(List= accRemoteScore) { // We skip vertices for which local calls outweigh the remote ones. continue; } + var totalAccScore = accRemoteScore - accLocalScore; var connVertices = ImmutableArray.CreateBuilder(); foreach (var edge in grainEdges) { @@ -632,7 +635,23 @@ private static HashSet ComputeAnchoredGrains(List accRemoteScore) { - anchoredGrains.Add(grainEdges.Key); + + + + + // anchoredGrains.Add(grainEdges.Key); + + + + + + + + + + + + } } @@ -740,6 +759,17 @@ void ISiloStatusListener.SiloStatusChangeNotification(SiloAddress updatedSilo, S _enableMessageSampling = _siloStatusOracle.GetActiveSilos().Length > 1; } + public ValueTask> GetGrainCallFrequencies() + { + var result = ImmutableArray.CreateBuilder<(Edge, ulong)>(_edgeWeights.Count); + foreach (var (edge, count, _) in _edgeWeights.Elements) + { + result.Add((edge, count)); + } + + return new(result.ToImmutable()); + } + private enum Direction : byte { Unspecified, diff --git a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs index 02fecd330d..ca6d0c7cb6 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/MaxHeap.cs @@ -96,6 +96,7 @@ public MaxHeap(List items) foreach (var item in items) { nodes[i] = item; + Debug.Assert(item.HeapIndex == -1); item.HeapIndex = i; i++; } diff --git a/test/Benchmarks/Ping/TreeGrain.cs b/test/Grains/BenchmarkGrains/Ping/TreeGrain.cs similarity index 90% rename from test/Benchmarks/Ping/TreeGrain.cs rename to test/Grains/BenchmarkGrains/Ping/TreeGrain.cs index 985e08ab04..fe8ff94708 100644 --- a/test/Benchmarks/Ping/TreeGrain.cs +++ b/test/Grains/BenchmarkGrains/Ping/TreeGrain.cs @@ -1,12 +1,12 @@ using BenchmarkGrainInterfaces.Ping; -namespace Benchmarks.Ping; +namespace BenchmarkGrains.Ping; public class TreeGrain : Grain, ITreeGrain { // 16^4 grains (~65K) - public const int FanOutFactor = 16; - public const int MaxLevel = 4; + public const int FanOutFactor = 4; + public const int MaxLevel = 3; private readonly List _children; public TreeGrain() diff --git a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs index 20b1917ace..df9f398113 100644 --- a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs @@ -18,6 +18,7 @@ public class CustomToleranceTests(CustomToleranceTests.Fixture fixture) : Rebala [Fact] public async Task Should_ConvertAllRemoteCalls_ToLocalCalls_WhileRespectingTolerance() { + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)); await AdjustActivationCountOffsets(); var e1 = GrainFactory.GetGrain(1); @@ -73,6 +74,7 @@ public async Task Should_ConvertAllRemoteCalls_ToLocalCalls_WhileRespectingToler e2_host = await e2.GetAddress(); e3_host = await e3.GetAddress(); f1_host = await f1.GetAddress(); + cts.Token.ThrowIfCancellationRequested(); } while (e2_host == Silo1 || e3_host == Silo1 || f1_host == Silo2); diff --git a/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs b/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs index af198ddbb5..0a590dde61 100644 --- a/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/MaxHeapTests.cs @@ -8,8 +8,8 @@ public sealed class MaxHeapTests public class MyHeapElement(int value) : IHeapElement { public int Value { get; set; } = value; - - public int HeapIndex { get; set; } + + public int HeapIndex { get; set; } = -1; public int CompareTo(MyHeapElement other) => Value.CompareTo(other.Value); public override string ToString() => $"{Value} @ {HeapIndex}"; From 6996c2cf44310e496cbb204ae77949d3715eff2d Mon Sep 17 00:00:00 2001 From: ReubenBond Date: Wed, 29 May 2024 09:33:38 -0700 Subject: [PATCH 24/28] fixups --- .../Rebalancing/ActivationRebalancer.cs | 18 +----------------- test/Grains/BenchmarkGrains/Ping/TreeGrain.cs | 4 ++-- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 199e4f872c..3a412313b9 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -635,23 +635,7 @@ private static HashSet ComputeAnchoredGrains(List accRemoteScore) { - - - - - // anchoredGrains.Add(grainEdges.Key); - - - - - - - - - - - - + anchoredGrains.Add(grainEdges.Key); } } diff --git a/test/Grains/BenchmarkGrains/Ping/TreeGrain.cs b/test/Grains/BenchmarkGrains/Ping/TreeGrain.cs index fe8ff94708..03b1c56c75 100644 --- a/test/Grains/BenchmarkGrains/Ping/TreeGrain.cs +++ b/test/Grains/BenchmarkGrains/Ping/TreeGrain.cs @@ -5,8 +5,8 @@ namespace BenchmarkGrains.Ping; public class TreeGrain : Grain, ITreeGrain { // 16^4 grains (~65K) - public const int FanOutFactor = 4; - public const int MaxLevel = 3; + public const int FanOutFactor = 16; + public const int MaxLevel = 4; private readonly List _children; public TreeGrain() From 4afc6b0e8468deb1398a7c463de1906ef202d8d0 Mon Sep 17 00:00:00 2001 From: Ledjon Behluli Date: Sun, 2 Jun 2024 23:50:59 +0200 Subject: [PATCH 25/28] Added faster variation of the bloom filter, and adjusted benchmarks --- .../ActivationRebalancer.MessageSink.cs | 2 +- .../Rebalancing/BlockedBloomFilter.cs | 125 +++++++++++++++ .../Placement/Rebalancing/BloomFilter.cs | 86 ---------- test/Benchmarks/TopK/BloomFilterBenchmark.cs | 147 +++++++++++++++++- 4 files changed, 265 insertions(+), 95 deletions(-) create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs delete mode 100644 src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index 0ca8d9e2b5..aeccc8a988 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -14,7 +14,7 @@ internal partial class ActivationRebalancer : IMessageStatisticsSink // This bloom filter contains grain ids which will are anchored to the current silo. // Ids are inserted when a grain is found to have a negative transfer score. - private readonly BloomFilter _anchoredGrainIds = new(100_000, 0.01); + private readonly BlockedBloomFilter _anchoredGrainIds = new(100_000, 0.01); private Task? _processPendingEdgesTask; public void StartProcessingEdges() diff --git a/src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs new file mode 100644 index 0000000000..9ae0244fb3 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs @@ -0,0 +1,125 @@ +using System; +using System.Diagnostics; +using System.IO.Hashing; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace Orleans.Runtime.Placement.Rebalancing; + +/// +/// A tuned version of a blocked bloom filter implementation. +/// +/// +/// This is a tuned version of BBF in order to meet the required FP rate. +/// Tuning takes a lot of time so this filter can accept FP rates in the rage of [0.1% - 1%] +/// Any value with the range, at any precision is supported as the FP rate is regressed via polynomial regression +/// More information can be read from Section 3: https://www.cs.amherst.edu/~ccmcgeoch/cs34/papers/cacheefficientbloomfilters-jea.pdf +/// +internal sealed class BlockedBloomFilter +{ + private const int BlockSize = 32; // higher value yields better speed, but at a high cost of space + private const double Ln2Squared = 0.4804530139182014246671025263266649717305529515945455; + private const double MinFpRate = 0.001; // 0.1% + private const double MaxFpRate = 0.01; // 1% + + private readonly int _blocks; + private readonly int[] _filter; + + // Regression coefficients (derived via polynomial regression) to match 'fpRate' as the actual deviates significantly with lower and lower 'fpRate' + private static readonly double[] _coefficients = { + 4.0102253166524500e-003, + -1.6272682781603145e+001, + 2.7169897602930665e+004, + -2.4527698904812500e+007, + 1.3273846004698063e+010, + -4.4943809759769805e+012, + 9.5588839677303638e+014, + -1.2081452101930328e+017, + 6.8958853188430172e+018, + 2.6889929911921561e+020, + -7.1061179529975569e+022, + 4.4109449793357217e+024, + -9.8041203512310751e+025 + }; + + /// The capcity to store. + /// Bounded within [ - ] + /// + public BlockedBloomFilter(int capacity, double fpRate) + { + if (fpRate < MinFpRate || fpRate > MaxFpRate) + { + throw new ArgumentOutOfRangeException($"False positive rate '{fpRate}', is outside of the allowed range '{MinFpRate} - {MaxFpRate}'"); + } + + double adjFpRate = RegressFpRate(fpRate); + Debug.Assert(adjFpRate < fpRate); + int bits = (int)((-1 * capacity * Math.Log(adjFpRate) / Ln2Squared)); + + _blocks = bits / BlockSize; + _filter = new int[_blocks + 1]; + } + + private static double RegressFpRate(double fpRate) + { + double temp = 1; + double result = 0; + + foreach (double coefficient in _coefficients) + { + result += coefficient * temp; + temp *= fpRate; + } + + return Math.Abs(result); + } + + public void Add(GrainId id) + { + ulong hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); + int index = GetBlockIndex(hash, _blocks); // important to get index before rotating the hash + + hash ^= BitOperations.RotateLeft(hash, 32); + + // We use 2 masks to distribute the bits of the hash value across multiple positions in the filter + int mask1 = ComputeMask1(hash); + int mask2 = ComputeMask2(hash); + + // We set the bits across 2 blocks so that the bits from a single hash value, are spread out more evenly across the filter. + _filter[index] |= mask1; + _filter[index + 1] |= mask2; + } + + public bool Contains(GrainId id) + { + ulong hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); + int index = GetBlockIndex(hash, _blocks); // important to get index before rotating the hash + + hash ^= BitOperations.RotateLeft(hash, 32); + + int block1 = _filter[index]; + int block2 = _filter[index + 1]; + + int mask1 = ComputeMask1(hash); + int mask2 = ComputeMask2(hash); + + return ((mask1 & block1) == mask1) && ((mask2 & block2) == mask2); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetBlockIndex(ulong hash, int buckets) => (int)(((int)hash & 0xffffffffL) * buckets >> 32); + + /// + /// Sets the bits of corresponding to the lower-order bits, and the bits shifted by 6 positions to the right + /// + /// The rotated hash + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeMask1(ulong hash) => (1 << (int)hash) | (1 << ((int)hash >> 6)); + + /// + /// Sets the bits of , and the bits shifted by 12 and 18 positions to the right + /// + /// The rotated hash + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int ComputeMask2(ulong hash) => (1 << ((int)hash >> 12)) | (1 << ((int)hash >> 18)); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs deleted file mode 100644 index 947ca35a29..0000000000 --- a/src/Orleans.Runtime/Placement/Rebalancing/BloomFilter.cs +++ /dev/null @@ -1,86 +0,0 @@ -using System; -using System.Diagnostics; -using System.IO.Hashing; -using System.Linq; -using System.Numerics; -using System.Runtime.CompilerServices; - -namespace Orleans.Runtime.Placement.Rebalancing; - -internal sealed class BloomFilter -{ - private const double Ln2Squared = 0.4804530139182014246671025263266649717305529515945455; - private const double Ln2 = 0.6931471805599453094172321214581765680755001343602552; - private readonly ulong[] _hashFuncSeeds; - private readonly int[] _filter; - private readonly int _indexMask; - - public BloomFilter(int capacity, double falsePositiveRate) - { - // Calculate the ideal bloom filter size and hash code count for the given (estimated) capacity and desired false positive rate. - // See https://en.wikipedia.org/wiki/Bloom_filter. - var minBitCount = (int)(-1 / Ln2Squared * capacity * Math.Log(falsePositiveRate)) / 8; - var arraySize = (int)CeilingPowerOfTwo((uint)(minBitCount - 1 + (1 << 5)) >> 5); - _indexMask = arraySize - 1; - _filter = new int[arraySize]; - - // Divide the hash count by 2 since we are using 64-bit hash codes split into two 32-bit hash codes. - var hashFuncCount = (int)Math.Min(minBitCount * 8 / capacity * Ln2 / 2, 8); - Debug.Assert(hashFuncCount > 0); - _hashFuncSeeds = Enumerable.Range(0, hashFuncCount).Select(p => unchecked((ulong)p * 0xFBA4C795FBA4C795 + 1)).ToArray(); - Debug.Assert(_hashFuncSeeds.Length == hashFuncCount); - } - - public void Add(GrainId id) - { - var hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); - foreach (var seed in _hashFuncSeeds) - { - hash = Mix64(hash ^ seed); - Set((int)hash); - Set((int)(hash >> 32)); - } - } - - public bool Contains(GrainId id) - { - var hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); - foreach (var seed in _hashFuncSeeds) - { - hash = Mix64(hash ^ seed); - var clear = IsClear((int)hash); - clear |= IsClear((int)(hash >> 32)); - if (clear) - { - return false; - } - } - - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool IsClear(int index) => (_filter[(index >> 5) & _indexMask] & (1 << index)) == 0; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Set(int index) => _filter[(index >> 5) & _indexMask] |= 1 << index; - - /// - /// Computes Stafford variant 13 of 64-bit mix function. - /// - /// The input parameter. - /// A bit mix of the input parameter. - /// - /// See http://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html - /// - public static ulong Mix64(ulong z) - { - z = (z ^ z >> 30) * 0xbf58476d1ce4e5b9L; - z = (z ^ z >> 27) * 0x94d049bb133111ebL; - return z ^ z >> 31; - } - - public void Reset() => Array.Clear(_filter); - - private static uint CeilingPowerOfTwo(uint x) => 1u << -BitOperations.LeadingZeroCount(x - 1); -} diff --git a/test/Benchmarks/TopK/BloomFilterBenchmark.cs b/test/Benchmarks/TopK/BloomFilterBenchmark.cs index 4b999d2c52..71298fa516 100644 --- a/test/Benchmarks/TopK/BloomFilterBenchmark.cs +++ b/test/Benchmarks/TopK/BloomFilterBenchmark.cs @@ -1,5 +1,8 @@ using System.Collections; +using System.Diagnostics; using System.IO.Hashing; +using System.Numerics; +using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; @@ -11,27 +14,33 @@ namespace Benchmarks.TopK; [MemoryDiagnoser] [FalsePositiveRateColumn] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory), CategoriesColumn] public class BloomFilterBenchmark { private BloomFilter _bloomFilter; private BloomFilter _bloomFilterWithSamples; private OriginalBloomFilter _originalBloomFilter; private OriginalBloomFilter _originalBloomFilterWithSamples; + private BlockedBloomFilter _blockedBloomFilter; + private BlockedBloomFilter _blockedBloomFilterWithSamples; private GrainId[] _population; private HashSet _set; private ZipfRejectionSampler _sampler; private GrainId[] _samples; - [Params(1_000_000, Priority = 3)] + [Params(1_000_000, Priority = 4)] public int Pop { get; set; } - [Params(/*0.2, 0.4, 0.6, 0.8, */1.02 /*, 1.2, 1.4, 1.6*/, Priority = 2)] + [Params(/*0.2, 0.4, 0.6, 0.8, */1.02 /*, 1.2, 1.4, 1.6*/, Priority = 3)] public double Skew { get; set; } [Params(1_000_000, Priority = 1)] public int Cap { get; set; } - [Params(10_000, Priority = 4)] + [Params(0.01, 0.001, Priority = 2)] + public double FP { get; set; } + + [Params(10_000, Priority = 5)] public int Samples { get; set; } [GlobalSetup] @@ -82,7 +91,6 @@ public void BloomFilter_Contains() } } - /* [Benchmark] [BenchmarkCategory("FP rate")] public int BloomFilter_FPR() @@ -103,9 +111,9 @@ public int BloomFilter_FPR() return incorrect; } - */ - [Benchmark] + + [Benchmark(Baseline = true)] [BenchmarkCategory("Add")] public void OriginalBloomFilter_Add() { @@ -115,7 +123,7 @@ public void OriginalBloomFilter_Add() } } - [Benchmark] + [Benchmark(Baseline = true)] [BenchmarkCategory("Contains")] public void OriginalBloomFilter_Contains() { @@ -126,7 +134,7 @@ public void OriginalBloomFilter_Contains() } /* - [Benchmark] + [Benchmark(Baseline = true)] [BenchmarkCategory("FP rate")] public int OriginalBloomFilter_FPR() { @@ -147,6 +155,50 @@ public int OriginalBloomFilter_FPR() return incorrect; } */ + + [Benchmark] + [BenchmarkCategory("Add")] + public void BlockedBloomFilter_Add() + { + foreach (var sample in _samples) + { + _blockedBloomFilter.Add(sample); + } + } + + [Benchmark] + [BenchmarkCategory("Contains")] + public void BlockedBloomFilter_Contains() + { + foreach (var sample in _samples) + { + _blockedBloomFilterWithSamples.Contains(sample); + } + } + + /* + // This is expected to yield a slighly higher FP rate, due to tuning + [Benchmark] + [BenchmarkCategory("FP rate")] + public int BlockedBloomFilter_FPR() + { + var correct = 0; + var incorrect = 0; + foreach (var sample in _population) + { + if (!_blockedBloomFilterWithSamples.Contains(sample) == _set.Contains(sample)) + { + correct++; + } + else + { + incorrect++; + } + } + + return incorrect; + } + */ } [AttributeUsage(AttributeTargets.Class)] @@ -194,3 +246,82 @@ public bool Contains(GrainId id) return true; } } + +internal sealed class BloomFilter +{ + private const double Ln2Squared = 0.4804530139182014246671025263266649717305529515945455; + private const double Ln2 = 0.6931471805599453094172321214581765680755001343602552; + private readonly ulong[] _hashFuncSeeds; + private readonly int[] _filter; + private readonly int _indexMask; + + public BloomFilter(int capacity, double falsePositiveRate) + { + // Calculate the ideal bloom filter size and hash code count for the given (estimated) capacity and desired false positive rate. + // See https://en.wikipedia.org/wiki/Bloom_filter. + var minBitCount = (int)(-1 / Ln2Squared * capacity * Math.Log(falsePositiveRate)) / 8; + var arraySize = (int)CeilingPowerOfTwo((uint)(minBitCount - 1 + (1 << 5)) >> 5); + _indexMask = arraySize - 1; + _filter = new int[arraySize]; + + // Divide the hash count by 2 since we are using 64-bit hash codes split into two 32-bit hash codes. + var hashFuncCount = (int)Math.Min(minBitCount * 8 / capacity * Ln2 / 2, 8); + Debug.Assert(hashFuncCount > 0); + _hashFuncSeeds = Enumerable.Range(0, hashFuncCount).Select(p => unchecked((ulong)p * 0xFBA4C795FBA4C795 + 1)).ToArray(); + Debug.Assert(_hashFuncSeeds.Length == hashFuncCount); + } + + public void Add(GrainId id) + { + var hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); + foreach (var seed in _hashFuncSeeds) + { + hash = Mix64(hash ^ seed); + Set((int)hash); + Set((int)(hash >> 32)); + } + } + + public bool Contains(GrainId id) + { + var hash = XxHash3.HashToUInt64(id.Key.AsSpan(), id.GetUniformHashCode()); + foreach (var seed in _hashFuncSeeds) + { + hash = Mix64(hash ^ seed); + var clear = IsClear((int)hash); + clear |= IsClear((int)(hash >> 32)); + if (clear) + { + return false; + } + } + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsClear(int index) => (_filter[(index >> 5) & _indexMask] & (1 << index)) == 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Set(int index) => _filter[(index >> 5) & _indexMask] |= 1 << index; + + /// + /// Computes Stafford variant 13 of 64-bit mix function. + /// + /// The input parameter. + /// A bit mix of the input parameter. + /// + /// See http://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html + /// + public static ulong Mix64(ulong z) + { + z = (z ^ z >> 30) * 0xbf58476d1ce4e5b9L; + z = (z ^ z >> 27) * 0x94d049bb133111ebL; + return z ^ z >> 31; + } + + public void Reset() => Array.Clear(_filter); + + private static uint CeilingPowerOfTwo(uint x) => 1u << -BitOperations.LeadingZeroCount(x - 1); +} + From 3f8b7e900f66af6325ac8f864f26f6d8bfe9d22f Mon Sep 17 00:00:00 2001 From: Ledjon Behluli Date: Sun, 9 Jun 2024 15:39:26 +0200 Subject: [PATCH 26/28] Introduced variable filtering options for anchored grains filtering --- .../Options/ActiveRebalancingOptions.cs | 29 +++++++++++++++++++ .../ActivationRebalancer.MessageSink.cs | 11 ++++--- .../Rebalancing/ActivationRebalancer.cs | 27 +++++++++++++---- .../Rebalancing/BlockedBloomFilter.cs | 4 ++- .../Rebalancing/IAnchoredGrainsFilter.cs | 8 +++++ 5 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 src/Orleans.Runtime/Placement/Rebalancing/IAnchoredGrainsFilter.cs diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index 4ee5a3ccbc..b3470d021d 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -72,6 +72,30 @@ public sealed class ActiveRebalancingOptions /// The default value of . /// public const int DEFAULT_MAX_UNPROCESSED_EDGES = 100_000; + + /// + /// When the algorithm has optimized partitioning, a lot of the edges will be internal to any given silo. + /// We track those edges to work out which activations should/shouldn't be transferred (cost vs benefit). + /// For any given activation, a lookup needs to happen and this flag controls wether that lookup should be + /// probabilistic in nature (there is a small error introduced inherently) or deterministic. + /// + public bool ProbabilisticFilteringEnabled { get; set; } = DEFAULT_PROBABILISTIC_FILTERING_ENABLED; + + /// + /// The default value of . + /// + public const bool DEFAULT_PROBABILISTIC_FILTERING_ENABLED = true; + + /// + /// The maximum allowed error rate when is set to , otherwise this does not apply. + /// + /// Allowed range: [0.001 - 0.01](0.1% - 1%) + public double ProbabilisticFilteringMaxAllowedErrorRate { get; set; } + + /// + /// The default value of . + /// + public double DEFAULT_PROBABILISTIC_FILTERING_MAX_ALLOWED_ERROR = 0.01d; } internal sealed class ActiveRebalancingOptionsValidator(IOptions options) : IConfigurationValidator @@ -114,6 +138,11 @@ public void ValidateConfiguration() { ThrowMustBeGreaterThanOrEqualTo(nameof(ActiveRebalancingOptions.MinRebalancingPeriod), nameof(ActiveRebalancingOptions.RecoveryPeriod)); } + + if (_options.ProbabilisticFilteringMaxAllowedErrorRate < 0.001d || _options.ProbabilisticFilteringMaxAllowedErrorRate > 0.01d) + { + throw new OrleansConfigurationException($"{nameof(ActiveRebalancingOptions.ProbabilisticFilteringMaxAllowedErrorRate)} must be inclusive between [0.001 - 0.01](0.1% - 1%)"); + } } private static void ThrowMustBeGreaterThanZero(string propertyName) diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs index aeccc8a988..b8b566fd4a 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.MessageSink.cs @@ -1,8 +1,11 @@ #nullable enable using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Orleans.Configuration; using Orleans.Placement.Rebalancing; using Orleans.Runtime.Internal; @@ -12,9 +15,9 @@ internal partial class ActivationRebalancer : IMessageStatisticsSink { private readonly CancellationTokenSource _shutdownCts = new(); - // This bloom filter contains grain ids which will are anchored to the current silo. + // This filter contains grain ids which will are anchored to the current silo. // Ids are inserted when a grain is found to have a negative transfer score. - private readonly BlockedBloomFilter _anchoredGrainIds = new(100_000, 0.01); + private readonly IAnchoredGrainsFilter _anchoredFilter; private Task? _processPendingEdgesTask; public void StartProcessingEdges() @@ -65,7 +68,7 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) } EdgeVertex sourceVertex; - if (_anchoredGrainIds.Contains(message.SendingGrain) && Silo.Equals(message.SendingSilo)) + if (_anchoredFilter.Contains(message.SendingGrain) && Silo.Equals(message.SendingSilo)) { sourceVertex = new(GrainId, Silo, isMigratable: false); } @@ -75,7 +78,7 @@ private async Task ProcessPendingEdges(CancellationToken cancellationToken) } EdgeVertex destinationVertex; - if (_anchoredGrainIds.Contains(message.TargetGrain) && Silo.Equals(message.TargetSilo)) + if (_anchoredFilter.Contains(message.TargetGrain) && Silo.Equals(message.TargetSilo)) { destinationVertex = new(GrainId, Silo, isMigratable: false); } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index 3a412313b9..c358dca77b 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -8,7 +8,6 @@ using System.Runtime.CompilerServices; using System.Diagnostics; using System.Collections.Immutable; -using Orleans.Core.Internal; using System.Data; using Orleans.Placement.Rebalancing; using System.Threading; @@ -65,9 +64,11 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe _migrationManager = migrationManager; _activationDirectory = activationDirectory; _timeProvider = timeProvider; - _edgeWeights = new((int)options.Value.MaxEdgeCount); + _edgeWeights = new(options.Value.MaxEdgeCount); _pendingMessages = new StripedMpscBuffer(Environment.ProcessorCount, options.Value.MaxUnprocessedEdges / Environment.ProcessorCount); - + _anchoredFilter = !options.Value.ProbabilisticFilteringEnabled ? new HashSetFilter() : + new BlockedBloomFilter(100_000, options.Value.ProbabilisticFilteringMaxAllowedErrorRate); + _lastExchangedStopwatch = CoarseStopwatch.StartNew(); catalog.RegisterSystemTarget(this); _siloStatusOracle.SubscribeToSiloStatusEvents(this); @@ -90,7 +91,7 @@ public ValueTask ResetCounters() { _pendingMessages.Clear(); _edgeWeights.Clear(); - _anchoredGrainIds.Reset(); + _anchoredFilter.Reset(); return ValueTask.CompletedTask; } @@ -485,7 +486,7 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr _logger.LogInformation("Adding {NewlyAnchoredGrains} newly anchored grains to set on host {Silo}. EdgeWeights contains {EdgeWeightCount} elements.", newlyAnchoredGrains.Count, Silo, _edgeWeights.Count); foreach (var id in newlyAnchoredGrains) { - _anchoredGrainIds.Add(id); + _anchoredFilter.Add(id); } foreach (var id in accepting) @@ -503,7 +504,7 @@ private async Task FinalizeProtocol(ImmutableArray giving, ImmutableArr { foreach (var (edge, _, _) in _edgeWeights.Elements) { - if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id) || _anchoredGrainIds.Contains(edge.Source.Id) || _anchoredGrainIds.Contains(edge.Target.Id)) + if (affected.Contains(edge.Source.Id) || affected.Contains(edge.Target.Id) || _anchoredFilter.Contains(edge.Source.Id) || _anchoredFilter.Contains(edge.Target.Id)) { toRemove.Add(edge); } @@ -772,4 +773,18 @@ private enum Direction : byte /// The edge's direction /// The number of estimated messages exchanged between and . private readonly record struct VertexEdge(GrainId SourceId, GrainId TargetId, bool IsMigratable, SiloAddress PartnerSilo, Direction Direction, long Weight); + + private class HashSetFilter : IAnchoredGrainsFilter + { + private readonly HashSet _hashSet = []; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Add(GrainId id) => _hashSet.Add(id); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Contains(GrainId id) => _hashSet.Contains(id); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Reset() => _hashSet.Clear(); + } } diff --git a/src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs index 9ae0244fb3..9dfe5d811e 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/BlockedBloomFilter.cs @@ -15,7 +15,7 @@ namespace Orleans.Runtime.Placement.Rebalancing; /// Any value with the range, at any precision is supported as the FP rate is regressed via polynomial regression /// More information can be read from Section 3: https://www.cs.amherst.edu/~ccmcgeoch/cs34/papers/cacheefficientbloomfilters-jea.pdf /// -internal sealed class BlockedBloomFilter +internal sealed class BlockedBloomFilter : IAnchoredGrainsFilter { private const int BlockSize = 32; // higher value yields better speed, but at a high cost of space private const double Ln2Squared = 0.4804530139182014246671025263266649717305529515945455; @@ -106,6 +106,8 @@ public bool Contains(GrainId id) return ((mask1 & block1) == mask1) && ((mask2 & block2) == mask2); } + public void Reset() => Array.Clear(_filter); + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int GetBlockIndex(ulong hash, int buckets) => (int)(((int)hash & 0xffffffffL) * buckets >> 32); diff --git a/src/Orleans.Runtime/Placement/Rebalancing/IAnchoredGrainsFilter.cs b/src/Orleans.Runtime/Placement/Rebalancing/IAnchoredGrainsFilter.cs new file mode 100644 index 0000000000..cf0e3d8560 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Rebalancing/IAnchoredGrainsFilter.cs @@ -0,0 +1,8 @@ +namespace Orleans.Runtime.Placement.Rebalancing; + +internal interface IAnchoredGrainsFilter +{ + void Add(GrainId id); + bool Contains(GrainId id); + void Reset(); +} \ No newline at end of file From 3458a73cd78c0af4d197323739c2f200d491d034 Mon Sep 17 00:00:00 2001 From: Ledjon Behluli Date: Wed, 19 Jun 2024 18:21:34 +0200 Subject: [PATCH 27/28] fixed some tests --- .../Options/ActiveRebalancingOptions.cs | 4 ++-- .../Rebalancing/ActivationRebalancer.cs | 5 ++-- ...terTests.cs => BlockedBloomFilterTests.cs} | 8 +++---- .../CustomToleranceTests.cs | 1 - .../ActiveRebalancingTests/OptionsTests.cs | 23 +++++++++++-------- 5 files changed, 23 insertions(+), 18 deletions(-) rename test/TesterInternal/ActiveRebalancingTests/{BloomFilterTests.cs => BlockedBloomFilterTests.cs} (70%) diff --git a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs index b3470d021d..a511559fac 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActiveRebalancingOptions.cs @@ -90,12 +90,12 @@ public sealed class ActiveRebalancingOptions /// The maximum allowed error rate when is set to , otherwise this does not apply. /// /// Allowed range: [0.001 - 0.01](0.1% - 1%) - public double ProbabilisticFilteringMaxAllowedErrorRate { get; set; } + public double ProbabilisticFilteringMaxAllowedErrorRate { get; set; } = DEFAULT_PROBABILISTIC_FILTERING_MAX_ALLOWED_ERROR; /// /// The default value of . /// - public double DEFAULT_PROBABILISTIC_FILTERING_MAX_ALLOWED_ERROR = 0.01d; + public const double DEFAULT_PROBABILISTIC_FILTERING_MAX_ALLOWED_ERROR = 0.01d; } internal sealed class ActiveRebalancingOptionsValidator(IOptions options) : IConfigurationValidator diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs index c358dca77b..ffb5b83e16 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancer.cs @@ -66,8 +66,9 @@ internal sealed partial class ActivationRebalancer : SystemTarget, IActivationRe _timeProvider = timeProvider; _edgeWeights = new(options.Value.MaxEdgeCount); _pendingMessages = new StripedMpscBuffer(Environment.ProcessorCount, options.Value.MaxUnprocessedEdges / Environment.ProcessorCount); - _anchoredFilter = !options.Value.ProbabilisticFilteringEnabled ? new HashSetFilter() : - new BlockedBloomFilter(100_000, options.Value.ProbabilisticFilteringMaxAllowedErrorRate); + _anchoredFilter = options.Value.ProbabilisticFilteringEnabled ? + new BlockedBloomFilter(100_000, options.Value.ProbabilisticFilteringMaxAllowedErrorRate) : + new HashSetFilter(); _lastExchangedStopwatch = CoarseStopwatch.StartNew(); catalog.RegisterSystemTarget(this); diff --git a/test/TesterInternal/ActiveRebalancingTests/BloomFilterTests.cs b/test/TesterInternal/ActiveRebalancingTests/BlockedBloomFilterTests.cs similarity index 70% rename from test/TesterInternal/ActiveRebalancingTests/BloomFilterTests.cs rename to test/TesterInternal/ActiveRebalancingTests/BlockedBloomFilterTests.cs index 20f0fc5a27..2909ae8e7a 100644 --- a/test/TesterInternal/ActiveRebalancingTests/BloomFilterTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/BlockedBloomFilterTests.cs @@ -1,14 +1,14 @@ -using Orleans.Runtime.Placement.Rebalancing; +using Orleans.Runtime.Placement.Rebalancing; using Xunit; namespace UnitTests.ActiveRebalancingTests; -public class BloomFilterTests +public class BlockedBloomFilterTests { [Fact] public void AddAndCheck() { - var bloomFilter = new BloomFilter(100, 0.01); + var bloomFilter = new BlockedBloomFilter(100, 0.01); var sample = new GrainId(GrainType.Create("type"), IdSpan.Create("key")); bloomFilter.Add(sample); Assert.True(bloomFilter.Contains(sample)); @@ -17,7 +17,7 @@ public void AddAndCheck() [Fact] public void DoesNotContainSome() { - var bloomFilter = new BloomFilter(100, 0.01); + var bloomFilter = new BlockedBloomFilter(100, 0.01); var sample = new GrainId(GrainType.Create("type"), IdSpan.Create("key")); Assert.False(bloomFilter.Contains(sample)); } diff --git a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs index df9f398113..65799a0eb2 100644 --- a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs @@ -3,7 +3,6 @@ using Orleans.Configuration; using Orleans.Placement; using Orleans.Placement.Rebalancing; -using Orleans.Runtime; using Orleans.Runtime.Placement; using Orleans.Runtime.Placement.Rebalancing; using Orleans.TestingHost; diff --git a/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs b/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs index e35e928d64..539b342887 100644 --- a/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/OptionsTests.cs @@ -11,6 +11,8 @@ public class OptionsTests [Fact] public void ConstantsShouldNotChange() { + Assert.True(ActiveRebalancingOptions.DEFAULT_PROBABILISTIC_FILTERING_ENABLED); + Assert.Equal(0.01d, ActiveRebalancingOptions.DEFAULT_PROBABILISTIC_FILTERING_MAX_ALLOWED_ERROR); Assert.Equal(10_000, ActiveRebalancingOptions.DEFAULT_MAX_EDGE_COUNT); Assert.Equal(TimeSpan.FromMinutes(1), ActiveRebalancingOptions.DEFAULT_MINUMUM_REBALANCING_PERIOD); Assert.Equal(TimeSpan.FromMinutes(2), ActiveRebalancingOptions.DEFAULT_MAXIMUM_REBALANCING_PERIOD); @@ -18,19 +20,21 @@ public void ConstantsShouldNotChange() } [Theory] - [InlineData(0, 1, 1, 1, 1)] - [InlineData(1, 0, 1, 1, 1)] - [InlineData(1, 1, 0, 1, 1)] - [InlineData(1, 1, 1, 0, 1)] - [InlineData(1, 1, 1, 1, 0)] - [InlineData(1, 1, 2, 1, 1)] - [InlineData(1, 1, 2, 1, 2)] + [InlineData(0, 1, 1, 1, 1, 0.01d)] + [InlineData(1, 0, 1, 1, 1, 0.01d)] + [InlineData(1, 1, 0, 1, 1, 0.01d)] + [InlineData(1, 1, 1, 0, 1, 0.01d)] + [InlineData(1, 1, 1, 1, 0, 0.01d)] + [InlineData(1, 1, 2, 1, 1, 0.01d)] + [InlineData(1, 1, 2, 1, 2, 0.01d)] + [InlineData(1, 1, 2, 1, 2, 0.1d)] public void InvalidOptionsShouldThrow( int topHeaviestCommunicationLinks, int maxUnprocessedEdges, int minRebalancingPeriodMinutes, int maxRebalancingPeriodMinutes, - int recoveryPeriodMinutes) + int recoveryPeriodMinutes, + double probabilisticFilteringMaxAllowedErrorRate) { var options = new ActiveRebalancingOptions { @@ -38,7 +42,8 @@ public void ConstantsShouldNotChange() MinRebalancingPeriod = TimeSpan.FromMinutes(minRebalancingPeriodMinutes), MaxRebalancingPeriod = TimeSpan.FromMinutes(maxRebalancingPeriodMinutes), RecoveryPeriod = TimeSpan.FromMinutes(recoveryPeriodMinutes), - MaxUnprocessedEdges = maxUnprocessedEdges + MaxUnprocessedEdges = maxUnprocessedEdges, + ProbabilisticFilteringMaxAllowedErrorRate = probabilisticFilteringMaxAllowedErrorRate }; var validator = new ActiveRebalancingOptionsValidator(Options.Create(options)); From caf32c6a8b6f7bfb5d77adb3dadb554be7cbc233 Mon Sep 17 00:00:00 2001 From: Ledjon Behluli Date: Wed, 19 Jun 2024 22:54:45 +0200 Subject: [PATCH 28/28] fixed all tests --- .../CustomToleranceTests.cs | 22 +++--- .../DefaultToleranceTests.cs | 72 +++---------------- 2 files changed, 20 insertions(+), 74 deletions(-) diff --git a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs index 65799a0eb2..ebed625cff 100644 --- a/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/CustomToleranceTests.cs @@ -11,13 +11,14 @@ namespace UnitTests.ActiveRebalancingTests; +// Scenarious can be seen visually here: https://github.com/dotnet/orleans/pull/8877 [TestCategory("Functional"), TestCategory("ActiveRebalancing"), Category("BVT")] public class CustomToleranceTests(CustomToleranceTests.Fixture fixture) : RebalancingTestBase(fixture), IClassFixture { [Fact] public async Task Should_ConvertAllRemoteCalls_ToLocalCalls_WhileRespectingTolerance() { - using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(1)); + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(10)); await AdjustActivationCountOffsets(); var e1 = GrainFactory.GetGrain(1); @@ -71,22 +72,21 @@ public async Task Should_ConvertAllRemoteCalls_ToLocalCalls_WhileRespectingToler do { e2_host = await e2.GetAddress(); - e3_host = await e3.GetAddress(); f1_host = await f1.GetAddress(); cts.Token.ThrowIfCancellationRequested(); } - while (e2_host == Silo1 || e3_host == Silo1 || f1_host == Silo2); - - await Test(); + while (e2_host == Silo1 || f1_host == Silo2); // At this point the layout is like follows: - // S1: E1-F1, sys.svc.clustering.dev, rest (default activations, present in both silos) - // S2: E2-F2, E3-F3, X, rest (default activations, present in both silos) + // S1: E1-F1, E3-F3, sys.svc.clustering.dev, rest (default activations, present in both silos) + // S2: E2-F2, X, rest (default activations, present in both silos) // Tolerance <= 2, and if we ignore defaults once, sys.svc.clustering.dev, and X (which is used to counter-balance sys.svc.clustering.dev) - // we end up with a total of 2 activations in silo1, and 4 in silo 2, which means the tolerance has been respected, and all remote calls have - // been converted to local calls: S1: E1-F1, S2: E2-F2, s2: E3-F3. + // we end up with a total of 4 activations in silo1, and 2 in silo 2, which means the tolerance has been respected, and all remote calls have + // been converted to local calls. + + await Test(); // To make sure, we trigger 's1_rebalancer' again, which should yield to no further migrations. i = 0; @@ -130,11 +130,11 @@ async Task Test() Assert.Equal(Silo1, e1_host); // E1 is still in silo 1 Assert.Equal(Silo2, e2_host); // E2 is now in silo 2 - Assert.Equal(Silo2, e3_host); // E3 is now in silo 2 + Assert.Equal(Silo1, e3_host); // E3 is still in silo 1 Assert.Equal(Silo1, f1_host); // F1 is now in silo 1 Assert.Equal(Silo2, f2_host); // F2 is still in silo 2 - Assert.Equal(Silo2, f3_host); // F3 is still in silo 2 + Assert.Equal(Silo1, f3_host); // F3 is now in silo 1 Assert.Equal(Silo2, await x.GetAddress()); // X remains in silo 2 } diff --git a/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs index 59cc082799..06f1ce821c 100644 --- a/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs +++ b/test/TesterInternal/ActiveRebalancingTests/DefaultToleranceTests.cs @@ -2,7 +2,6 @@ using Microsoft.Extensions.DependencyInjection; using Orleans.Configuration; using Orleans.Placement; -using Orleans.Runtime; using Orleans.Runtime.Placement; using Orleans.Runtime.Placement.Rebalancing; using Orleans.Streams; @@ -12,6 +11,7 @@ namespace UnitTests.ActiveRebalancingTests; +// Scenarious can be seen visually here: https://github.com/dotnet/orleans/pull/8877 [TestCategory("Functional"), TestCategory("ActiveRebalancing")] public class DefaultToleranceTests(DefaultToleranceTests.Fixture fixture) : RebalancingTestBase(fixture), IClassFixture { @@ -158,13 +158,13 @@ public async Task Immovable_C_ShouldStayOnSilo2__A_And_B_ShouldMoveToSilo2() Assert.Equal(Silo2, a_host); // A is now in silo 2 Assert.Equal(Silo2, b_host); // B is now in silo 2 - Assert.Equal(Silo2, c_host); + Assert.Equal(Silo2, c_host); // C is still in silo 2 await ResetCounters(); } [Fact] - public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOtherWayAround() + public async Task A_ShouldMoveToSilo2_Or_C_ShouldMoveToSilo1__B_And_D_ShouldStayOnTheirSilos() { RequestContext.Set(IPlacementDirector.PlacementHintKey, Silo1); @@ -173,9 +173,11 @@ public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOthe await a.FirstPing(scenario, Silo1, Silo2); - for (var i = 0; i < 3; ++i) + var i = 0; + while (i < 3) { await a.Ping(scenario); + i++; } var b = GrainFactory.GetGrain($"b{scenario}"); @@ -192,7 +194,6 @@ public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOthe Assert.Equal(Silo2, c_host); Assert.Equal(Silo2, d_host); - // 1st cycle await Silo1Rebalancer.TriggerExchangeRequest(); do @@ -200,7 +201,7 @@ public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOthe a_host = await a.GetAddress(); c_host = await c.GetAddress(); } - while (a_host != c_host); + while (a_host == Silo1 && c_host == Silo2); // refresh a_host = await a.GetAddress(); @@ -208,42 +209,14 @@ public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOthe c_host = await c.GetAddress(); d_host = await d.GetAddress(); - // A can go to Silo 2, or C can come to Silo 1, both are valid, so we need to check for both! + // A can go to Silo 2, or C can come to Silo 1, both are valid, so we need to check for both. if (a_host == Silo2 && c_host == Silo2) { - // A is now in silo 2 - Assert.Equal(Silo2, a_host); + Assert.Equal(Silo2, a_host); // A is now in silo 2 Assert.Equal(Silo1, b_host); Assert.Equal(Silo2, c_host); Assert.Equal(Silo2, d_host); - // 2nd cycle - for (var i = 0; i < 3; i++) - { - await a.Ping(scenario); - } - - // Since A moved to silo 2 at this point, it will be twice as strongly connected to C as it is to B, - // even though its now making remote calls (to B)! That's why we trigger the exchange from 's1_rebalancer' - await Silo1Rebalancer.TriggerExchangeRequest(); - - do - { - b_host = await b.GetAddress(); - } - while (b_host == Silo1); - - // refresh - a_host = await a.GetAddress(); - b_host = await b.GetAddress(); - c_host = await c.GetAddress(); - d_host = await d.GetAddress(); - - Assert.Equal(Silo2, a_host); - Assert.Equal(Silo2, b_host); // B is now in silo 2 - Assert.Equal(Silo2, c_host); - Assert.Equal(Silo2, d_host); - return; } @@ -254,33 +227,6 @@ public async Task A_And_B_ShouldMoveToSilo2__C_And_D_ShouldStayOnSilo2_OrTheOthe Assert.Equal(Silo1, c_host); // C is now in silo 1 Assert.Equal(Silo2, d_host); - // 2nd cycle - for (var i = 0; i < 3; i++) - { - await a.Ping(scenario); - } - - // Since C moved to silo 1 at this point, it will be twice as strongly connected to A as it is to D, - // even though its now making remote calls (to D)! Thats why we trigger the exchange from 's2_rebalancer' - await Silo2Rebalancer.TriggerExchangeRequest(); - - do - { - d_host = await b.GetAddress(); - } - while (d_host == Silo2); - - // refresh - a_host = await a.GetAddress(); - b_host = await b.GetAddress(); - c_host = await c.GetAddress(); - d_host = await d.GetAddress(); - - Assert.Equal(Silo1, a_host); - Assert.Equal(Silo1, b_host); - Assert.Equal(Silo1, c_host); - Assert.Equal(Silo1, d_host); // D is now in silo 1 - return; }