From 7ba587db7df8e9d61bb3894a172eb44a92f742fc Mon Sep 17 00:00:00 2001 From: Ismael Hamed <1279846+ismaelhamed@users.noreply.github.com> Date: Wed, 18 Mar 2020 20:35:01 +0100 Subject: [PATCH] Add circuit breaker exponential backoff support --- docs/articles/utilities/circuit-breaker.md | 30 +++-- .../CoreAPISpec.ApproveCore.approved.txt | 14 ++- .../Utilities/CircuitBreakerDocSpec.cs | 17 +-- .../Akka.Tests/Pattern/CircuitBreakerSpec.cs | 63 ++++++++-- src/core/Akka/Pattern/CircuitBreaker.cs | 112 ++++++++++++++---- src/core/Akka/Pattern/CircuitBreakerState.cs | 43 +++++-- src/core/Akka/Pattern/OpenCircuitException.cs | 29 ++++- 7 files changed, 230 insertions(+), 78 deletions(-) diff --git a/docs/articles/utilities/circuit-breaker.md b/docs/articles/utilities/circuit-breaker.md index 669dd3c72a9..f8cc9efd1d8 100644 --- a/docs/articles/utilities/circuit-breaker.md +++ b/docs/articles/utilities/circuit-breaker.md @@ -16,25 +16,23 @@ The Akka.NET library provides an implementation of a circuit breaker called `Akk ## What do they do? * During normal operation, a circuit breaker is in the `Closed` state: - * Exceptions or calls exceeding the configured `СallTimeout` increment a - failure counter - * Successes reset the failure count to zero - * When the failure counter reaches a `MaxFailures` count, the breaker is - tripped into `Open` state + * Exceptions or calls exceeding the configured `СallTimeout` increment a failure counter + * Successes reset the failure count to zero + * When the failure counter reaches a `MaxFailures` count, the breaker is tripped into `Open` state + * While in `Open` state: - * All calls fail-fast with a `OpenCircuitException` - * After the configured `ResetTimeout`, the circuit breaker enters a - `Half-Open` state + * All calls fail-fast with a `OpenCircuitException` + * After the configured `ResetTimeout`, the circuit breaker enters a `Half-Open` state + * In `Half-Open` state: - * The first call attempted is allowed through without failing fast - * All other calls fail-fast with an exception just as in `Open` state - * If the first call succeeds, the breaker is reset back to `Closed` state - * If the first call fails, the breaker is tripped again into the `Open` state - for another full `ResetTimeout` + * The first call attempted is allowed through without failing fast + * All other calls fail-fast with an exception just as in `Open` state + * If the first call succeeds, the breaker is reset back to `Closed` state and the `ResetTimeout` is reset + * If the first call fails, the breaker is tripped again into the `Open` state (as for exponential backoff circuit breaker, the `ResetTimeout` is multiplied by the exponential backoff factor) + * State transition listeners: - * Callbacks can be provided for every state entry via `OnOpen`, `OnClose`, - and `OnHalfOpen` - * These are executed in the `ExecutionContext` provided. + * Callbacks can be provided for every state entry via `OnOpen`, `OnClose`, and `OnHalfOpen` + * These are executed in the `ExecutionContext` provided. ![Circuit breaker states](/images/circuit-breaker-states.png) diff --git a/src/core/Akka.API.Tests/CoreAPISpec.ApproveCore.approved.txt b/src/core/Akka.API.Tests/CoreAPISpec.ApproveCore.approved.txt index ef3f1302f11..3ca071524a9 100644 --- a/src/core/Akka.API.Tests/CoreAPISpec.ApproveCore.approved.txt +++ b/src/core/Akka.API.Tests/CoreAPISpec.ApproveCore.approved.txt @@ -3810,15 +3810,19 @@ namespace Akka.Pattern } public class CircuitBreaker { - public CircuitBreaker(int maxFailures, System.TimeSpan callTimeout, System.TimeSpan resetTimeout) { } + public CircuitBreaker(Akka.Actor.IScheduler scheduler, int maxFailures, System.TimeSpan callTimeout, System.TimeSpan resetTimeout) { } + public CircuitBreaker(Akka.Actor.IScheduler scheduler, int maxFailures, System.TimeSpan callTimeout, System.TimeSpan resetTimeout, System.TimeSpan maxResetTimeout, double exponentialBackoffFactor) { } public System.TimeSpan CallTimeout { get; } public long CurrentFailureCount { get; } + public double ExponentialBackoffFactor { get; } public bool IsClosed { get; } public bool IsHalfOpen { get; } public bool IsOpen { get; } public int MaxFailures { get; } + public System.TimeSpan MaxResetTimeout { get; } public System.TimeSpan ResetTimeout { get; } - public static Akka.Pattern.CircuitBreaker Create(int maxFailures, System.TimeSpan callTimeout, System.TimeSpan resetTimeout) { } + public Akka.Actor.IScheduler Scheduler { get; } + public static Akka.Pattern.CircuitBreaker Create(Akka.Actor.IScheduler scheduler, int maxFailures, System.TimeSpan callTimeout, System.TimeSpan resetTimeout) { } public void Fail() { } public Akka.Pattern.CircuitBreaker OnClose(System.Action callback) { } public Akka.Pattern.CircuitBreaker OnHalfOpen(System.Action callback) { } @@ -3826,6 +3830,7 @@ namespace Akka.Pattern public void Succeed() { } public System.Threading.Tasks.Task WithCircuitBreaker(System.Func> body) { } public System.Threading.Tasks.Task WithCircuitBreaker(System.Func body) { } + public Akka.Pattern.CircuitBreaker WithExponentialBackoff(System.TimeSpan maxResetTimeout) { } public void WithSyncCircuitBreaker(System.Action body) { } public T WithSyncCircuitBreaker(System.Func body) { } } @@ -3841,9 +3846,12 @@ namespace Akka.Pattern } public class OpenCircuitException : Akka.Actor.AkkaException { - public OpenCircuitException() { } + public OpenCircuitException(System.TimeSpan remainingDuration) { } public OpenCircuitException(string message) { } + public OpenCircuitException(string message, System.TimeSpan remainingDuration) { } public OpenCircuitException(string message, System.Exception cause) { } + public OpenCircuitException(string message, System.Exception cause, System.TimeSpan remainingDuration) { } + public System.TimeSpan RemainingDuration { get; } } } namespace Akka.Routing diff --git a/src/core/Akka.Docs.Tests/Utilities/CircuitBreakerDocSpec.cs b/src/core/Akka.Docs.Tests/Utilities/CircuitBreakerDocSpec.cs index 70a183f748a..10e6b4cb0d6 100644 --- a/src/core/Akka.Docs.Tests/Utilities/CircuitBreakerDocSpec.cs +++ b/src/core/Akka.Docs.Tests/Utilities/CircuitBreakerDocSpec.cs @@ -21,10 +21,10 @@ public class DangerousActor : ReceiveActor public DangerousActor() { var breaker = new CircuitBreaker( - maxFailures: 5, - callTimeout: TimeSpan.FromSeconds(10), - resetTimeout: TimeSpan.FromMinutes(1)) - .OnOpen(NotifyMeOnOpen); + Context.System.Scheduler, + maxFailures: 5, + callTimeout: TimeSpan.FromSeconds(10), + resetTimeout: TimeSpan.FromMinutes(1)).OnOpen(NotifyMeOnOpen); } private void NotifyMeOnOpen() @@ -42,10 +42,10 @@ public class DangerousActorCallProtection : ReceiveActor public DangerousActorCallProtection() { var breaker = new CircuitBreaker( - maxFailures: 5, - callTimeout: TimeSpan.FromSeconds(10), - resetTimeout: TimeSpan.FromMinutes(1)) - .OnOpen(NotifyMeOnOpen); + Context.System.Scheduler, + maxFailures: 5, + callTimeout: TimeSpan.FromSeconds(10), + resetTimeout: TimeSpan.FromMinutes(1)).OnOpen(NotifyMeOnOpen); var dangerousCall = "This really isn't that dangerous of a call after all"; @@ -77,6 +77,7 @@ public TellPatternActor(IActorRef recipient ) { _recipient = recipient; _breaker = new CircuitBreaker( + Context.System.Scheduler, maxFailures: 5, callTimeout: TimeSpan.FromSeconds(10), resetTimeout: TimeSpan.FromMinutes(1)).OnOpen(NotifyMeOnOpen); diff --git a/src/core/Akka.Tests/Pattern/CircuitBreakerSpec.cs b/src/core/Akka.Tests/Pattern/CircuitBreakerSpec.cs index 0ad0e4eb044..3150c1258b4 100644 --- a/src/core/Akka.Tests/Pattern/CircuitBreakerSpec.cs +++ b/src/core/Akka.Tests/Pattern/CircuitBreakerSpec.cs @@ -6,6 +6,7 @@ //----------------------------------------------------------------------- using System; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.Serialization; @@ -262,7 +263,6 @@ public void Should_Pass_Call_And_Transition_To_Open_On_Exception( ) { var breaker = ShortResetTimeoutCb( ); - Assert.True( InterceptExceptionType( ( ) => breaker.Instance.WithCircuitBreaker( () => Task.Factory.StartNew( ThrowException ) ).Wait( ) ) ); Assert.True( CheckLatch( breaker.HalfOpenLatch ) ); @@ -303,6 +303,29 @@ public void Should_Transition_To_Half_Open_When_Reset_Timeout( ) Assert.True( InterceptExceptionType( ( ) => breaker.Instance.WithCircuitBreaker( () => Task.Factory.StartNew( ThrowException ) ).Wait( ) ) ); Assert.True( CheckLatch( breaker.HalfOpenLatch ) ); } + + [Fact(DisplayName = "An asynchronous circuit breaker that is open should increase the reset timeout after it transits to open again")] + public void Should_Reset_Timeout_After_It_Transits_To_Open_Again() + { + var breaker = NonOneFactorCb(); + Assert.True(InterceptExceptionType(() => breaker.Instance.WithCircuitBreaker(() => Task.Run(ThrowException)).Wait())); + Assert.True(CheckLatch(breaker.OpenLatch)); + + var e1 = InterceptException(() => breaker.Instance.WithSyncCircuitBreaker(SayTest)); + var shortRemainingDuration = e1.RemainingDuration; + + Thread.Sleep(1000); + Assert.True(CheckLatch(breaker.HalfOpenLatch)); + + // transit to open again + Assert.True(InterceptExceptionType(() => breaker.Instance.WithCircuitBreaker(() => Task.Run(ThrowException)).Wait())); + Assert.True(CheckLatch(breaker.OpenLatch)); + + var e2 = InterceptException(() => breaker.Instance.WithSyncCircuitBreaker(SayTest)); + var longRemainingDuration = e2.RemainingDuration; + + Assert.True(shortRemainingDuration < longRemainingDuration); + } } public class CircuitBreakerSpecBase : AkkaSpec @@ -320,14 +343,25 @@ public Task Delay( TimeSpan toDelay, CancellationToken? token ) return token.HasValue ? Task.Delay( toDelay, token.Value ) : Task.Delay( toDelay ); } - public void ThrowException( ) - { - throw new TestException( "Test Exception" ); - } + [DebuggerStepThrough] + public void ThrowException() => throw new TestException("Test Exception"); + + public string SayTest() => "Test"; - public string SayTest( ) + protected T InterceptException(Action actionThatThrows) where T : Exception { - return "Test"; + return Assert.Throws(() => + { + try + { + actionThatThrows(); + } + catch (AggregateException ex) + { + foreach (var e in ex.Flatten().InnerExceptions.Where(e => e is T).Select(e => e)) + throw e; + } + }); } [SuppressMessage( "Microsoft.Design", "CA1004:GenericMethodsShouldProvideTypeParameter" )] @@ -366,27 +400,32 @@ public string SayTest( ) public TestBreaker ShortCallTimeoutCb( ) { - return new TestBreaker( new CircuitBreaker( 1, TimeSpan.FromMilliseconds( 50 ), TimeSpan.FromMilliseconds( 500 ) ) ); + return new TestBreaker( new CircuitBreaker(Sys.Scheduler, 1, TimeSpan.FromMilliseconds( 50 ), TimeSpan.FromMilliseconds( 500 ) ) ); } public TestBreaker ShortResetTimeoutCb( ) { - return new TestBreaker( new CircuitBreaker( 1, TimeSpan.FromMilliseconds( 1000 ), TimeSpan.FromMilliseconds( 50 ) ) ); + return new TestBreaker( new CircuitBreaker(Sys.Scheduler, 1, TimeSpan.FromMilliseconds( 1000 ), TimeSpan.FromMilliseconds( 50 ) ) ); } public TestBreaker LongCallTimeoutCb( ) { - return new TestBreaker( new CircuitBreaker( 1, TimeSpan.FromMilliseconds( 5000 ), TimeSpan.FromMilliseconds( 500 ) ) ); + return new TestBreaker( new CircuitBreaker(Sys.Scheduler, 1, TimeSpan.FromMilliseconds( 5000 ), TimeSpan.FromMilliseconds( 500 ) ) ); } public TestBreaker LongResetTimeoutCb( ) { - return new TestBreaker( new CircuitBreaker( 1, TimeSpan.FromMilliseconds( 100 ), TimeSpan.FromMilliseconds( 5000 ) ) ); + return new TestBreaker( new CircuitBreaker(Sys.Scheduler, 1, TimeSpan.FromMilliseconds( 100 ), TimeSpan.FromMilliseconds( 5000 ) ) ); } public TestBreaker MultiFailureCb( ) { - return new TestBreaker( new CircuitBreaker( 5, TimeSpan.FromMilliseconds( 200 ), TimeSpan.FromMilliseconds( 500 ) ) ); + return new TestBreaker( new CircuitBreaker(Sys.Scheduler, 5, TimeSpan.FromMilliseconds( 200 ), TimeSpan.FromMilliseconds( 500 ) ) ); + } + + public TestBreaker NonOneFactorCb() + { + return new TestBreaker(new CircuitBreaker(Sys.Scheduler, 1, TimeSpan.FromMilliseconds(2000), TimeSpan.FromMilliseconds(1000), TimeSpan.FromDays(1), 5)); } } diff --git a/src/core/Akka/Pattern/CircuitBreaker.cs b/src/core/Akka/Pattern/CircuitBreaker.cs index ddc6aba8362..401cc6b9484 100644 --- a/src/core/Akka/Pattern/CircuitBreaker.cs +++ b/src/core/Akka/Pattern/CircuitBreaker.cs @@ -10,6 +10,7 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; +using Akka.Actor; using Akka.Util.Internal; namespace Akka.Pattern @@ -49,6 +50,11 @@ public class CircuitBreaker /// private AtomicState _currentState; + /// + /// Holds reference to current resetTimeout of CircuitBreaker - *access only via helper methods* + /// + private long _currentResetTimeout; + /// /// Helper method for access to the underlying state via Interlocked /// @@ -72,19 +78,55 @@ private AtomicState CurrentState } } + /// + /// Helper method for updating the underlying resetTimeout via Interlocked + /// + internal bool SwapStateResetTimeout(TimeSpan oldResetTimeout, TimeSpan newResetTimeout) + { + return Interlocked.CompareExchange(ref _currentResetTimeout, newResetTimeout.Ticks, oldResetTimeout.Ticks) == oldResetTimeout.Ticks; + } + + /// + /// Helper method for access to the underlying resetTimeout via Interlocked + /// + internal TimeSpan CurrentResetTimeout + { + get + { + Interlocked.MemoryBarrier(); + return TimeSpan.FromTicks(_currentResetTimeout); + } + } + + /// + /// TBD + /// + public IScheduler Scheduler { get; } + + /// + /// TBD + /// + public int MaxFailures { get; } + + /// + /// TBD + /// + public TimeSpan CallTimeout { get; } + /// /// TBD /// - public int MaxFailures { get; private set; } + public TimeSpan ResetTimeout { get; } /// /// TBD /// - public TimeSpan CallTimeout { get; private set; } + public TimeSpan MaxResetTimeout { get; } + /// /// TBD /// - public TimeSpan ResetTimeout { get; private set; } + public double ExponentialBackoffFactor { get; } //akka.io implementation is to use nested static classes and access parent member variables //.Net static nested classes do not have access to parent member variables -- so we configure the states here and @@ -96,41 +138,60 @@ private AtomicState CurrentState /// /// Create a new CircuitBreaker /// + /// Reference to Akka scheduler + /// Maximum number of failures before opening the circuit + /// of time after which to consider a call a failure + /// of time after which to attempt to close the circuit + /// TBD + public static CircuitBreaker Create(IScheduler scheduler, int maxFailures, TimeSpan callTimeout, TimeSpan resetTimeout) + { + return new CircuitBreaker(scheduler, maxFailures, callTimeout, resetTimeout); + } + + /// + /// Create a new CircuitBreaker + /// + /// Reference to Akka scheduler /// Maximum number of failures before opening the circuit /// of time after which to consider a call a failure /// of time after which to attempt to close the circuit /// TBD - public static CircuitBreaker Create(int maxFailures, TimeSpan callTimeout, TimeSpan resetTimeout) + public CircuitBreaker(IScheduler scheduler, int maxFailures, TimeSpan callTimeout, TimeSpan resetTimeout) + : this(scheduler, maxFailures, callTimeout, resetTimeout, TimeSpan.FromDays(36500), 1.0) { - return new CircuitBreaker(maxFailures, callTimeout, resetTimeout); } /// /// Create a new CircuitBreaker /// + /// Reference to Akka scheduler /// Maximum number of failures before opening the circuit /// of time after which to consider a call a failure /// of time after which to attempt to close the circuit + /// + /// /// TBD - public CircuitBreaker(int maxFailures, TimeSpan callTimeout, TimeSpan resetTimeout) + public CircuitBreaker(IScheduler scheduler, int maxFailures, TimeSpan callTimeout, TimeSpan resetTimeout, TimeSpan maxResetTimeout, double exponentialBackoffFactor) { + if (exponentialBackoffFactor < 1.0) throw new ArgumentException("factor must be >= 1.0", nameof(exponentialBackoffFactor)); + + Scheduler = scheduler; MaxFailures = maxFailures; CallTimeout = callTimeout; ResetTimeout = resetTimeout; + MaxResetTimeout = maxResetTimeout; + ExponentialBackoffFactor = exponentialBackoffFactor; Closed = new Closed(this); Open = new Open(this); HalfOpen = new HalfOpen(this); _currentState = Closed; - //_failures = new AtomicInteger(); + _currentResetTimeout = resetTimeout.Ticks; } /// /// Retrieves current failure count. /// - public long CurrentFailureCount - { - get { return Closed.Current; } - } + public long CurrentFailureCount => Closed.Current; /// /// Wraps invocation of asynchronous calls that need to be protected @@ -140,7 +201,7 @@ public long CurrentFailureCount /// containing the call result public Task WithCircuitBreaker(Func> body) { - return CurrentState.Invoke(body); + return CurrentState.Invoke(body); } /// @@ -261,6 +322,16 @@ public CircuitBreaker OnClose(Action callback) return this; } + /// + /// The will be increased exponentially for each failed attempt to close the circuit. + /// The default exponential backoff factor is 2. + /// + /// The upper bound of + public CircuitBreaker WithExponentialBackoff(TimeSpan maxResetTimeout) + { + return new CircuitBreaker(Scheduler, MaxFailures, CallTimeout, ResetTimeout, maxResetTimeout, 2.0); + } + /// /// Implements consistent transition between states. Throws IllegalStateException if an invalid transition is attempted. /// @@ -283,27 +354,16 @@ private void Transition(AtomicState fromState, AtomicState toState) /// Trips breaker to an open state. This is valid from Closed or Half-Open states /// /// State we're coming from (Closed or Half-Open) - internal void TripBreaker(AtomicState fromState) - { - Transition(fromState, Open); - } + internal void TripBreaker(AtomicState fromState) => Transition(fromState, Open); /// /// Resets breaker to a closed state. This is valid from an Half-Open state only. /// - internal void ResetBreaker() - { - Transition(HalfOpen, Closed); - } + internal void ResetBreaker() => Transition(HalfOpen, Closed); /// /// Attempts to reset breaker by transitioning to a half-open state. This is valid from an Open state only. /// - internal void AttemptReset() - { - Transition(Open, HalfOpen); - } - - //private readonly Task timeoutTask = Task.FromResult(new TimeoutException("Circuit Breaker Timed out.")); + internal void AttemptReset() => Transition(Open, HalfOpen); } } diff --git a/src/core/Akka/Pattern/CircuitBreakerState.cs b/src/core/Akka/Pattern/CircuitBreakerState.cs index 53064bce917..083bce0bf89 100644 --- a/src/core/Akka/Pattern/CircuitBreakerState.cs +++ b/src/core/Akka/Pattern/CircuitBreakerState.cs @@ -31,26 +31,37 @@ public Open(CircuitBreaker breaker) } /// - /// N/A + /// Calculate remaining duration until reset to inform the caller in case a backoff algorithm is useful + /// + /// Duration to when the breaker will attempt a reset by transitioning to half-open + private TimeSpan RemainingDuration() + { + var fromOpened = DateTime.UtcNow.Ticks - Current; + var diff = _breaker.CurrentResetTimeout.Ticks - fromOpened; + return diff <= 0L ? TimeSpan.Zero : TimeSpan.FromTicks(diff); + } + + /// + /// Fail-fast on any invocation /// /// N/A - /// N/A + /// Implementation of the call that needs protected /// This exception is thrown automatically since the circuit is open. /// N/A public override async Task Invoke(Func> body) { - throw new OpenCircuitException(); + throw new OpenCircuitException(RemainingDuration()); } /// - /// N/A + /// Fail-fast on any invocation /// - /// N/A + /// Implementation of the call that needs protected /// This exception is thrown automatically since the circuit is open. /// N/A public override async Task Invoke(Func body) { - throw new OpenCircuitException(); + throw new OpenCircuitException(RemainingDuration()); } /// @@ -68,12 +79,19 @@ protected internal override void CallSucceeds() } /// - /// On entering this state, schedule an attempted reset and store the entry time to + /// On entering this state, schedule an attempted reset via and store the entry time to /// calculate remaining time before attempted reset. /// protected override void EnterInternal() { - Task.Delay(_breaker.ResetTimeout).ContinueWith(task => _breaker.AttemptReset()); + GetAndSet(DateTime.UtcNow.Ticks); + _breaker.Scheduler.Advanced.ScheduleOnce(_breaker.CurrentResetTimeout, () => _breaker.AttemptReset()); + + var nextResetTimeout = TimeSpan.FromTicks(_breaker.CurrentResetTimeout.Ticks * (long)_breaker.ExponentialBackoffFactor); + if (nextResetTimeout < _breaker.MaxResetTimeout) + { + _breaker.SwapStateResetTimeout(_breaker.CurrentResetTimeout, nextResetTimeout); + } } /// @@ -113,7 +131,7 @@ public override async Task Invoke(Func> body) { if (!_lock.CompareAndSet(true, false)) { - throw new OpenCircuitException(); + throw new OpenCircuitException(TimeSpan.Zero); } return await CallThrough(body); } @@ -129,7 +147,7 @@ public override async Task Invoke(Func body) { if (!_lock.CompareAndSet(true, false)) { - throw new OpenCircuitException(); + throw new OpenCircuitException(TimeSpan.Zero); } await CallThrough(body); } @@ -232,13 +250,14 @@ protected internal override void CallSucceeds() protected override void EnterInternal() { Reset(); + _breaker.SwapStateResetTimeout(_breaker.CurrentResetTimeout, _breaker.ResetTimeout); } /// - /// Returns a that represents this instance. + /// Returns a that represents this instance. /// /// - /// A that represents this instance. + /// A that represents this instance. /// public override string ToString() { diff --git a/src/core/Akka/Pattern/OpenCircuitException.cs b/src/core/Akka/Pattern/OpenCircuitException.cs index cc687870096..dd2212fcd68 100644 --- a/src/core/Akka/Pattern/OpenCircuitException.cs +++ b/src/core/Akka/Pattern/OpenCircuitException.cs @@ -16,18 +16,33 @@ namespace Akka.Pattern /// public class OpenCircuitException : AkkaException { + public TimeSpan RemainingDuration { get; } + /// /// Initializes a new instance of the class. /// - public OpenCircuitException() : base("Circuit Breaker is open; calls are failing fast") { } + /// Stores remaining time before attempting a reset. Zero duration means the breaker is currently in half-open state + public OpenCircuitException(TimeSpan remainingDuration) + : this("Circuit Breaker is open; calls are failing fast", remainingDuration) + { } /// /// Initializes a new instance of the class. /// /// The message that describes the error. public OpenCircuitException(string message) + : this(message, TimeSpan.Zero) + { } + + /// + /// Initializes a new instance of the class. + /// + /// The message that describes the error. + /// Stores remaining time before attempting a reset. Zero duration means the breaker is currently in half-open state + public OpenCircuitException(string message, TimeSpan remainingDuration) : base(message) { + RemainingDuration = remainingDuration; } /// @@ -36,8 +51,20 @@ public OpenCircuitException(string message) /// The message that describes the error. /// The exception that is the cause of the current exception. public OpenCircuitException(string message, Exception cause) + : this(message, cause, TimeSpan.Zero) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The message that describes the error. + /// The exception that is the cause of the current exception. + /// Stores remaining time before attempting a reset. Zero duration means the breaker is currently in half-open state + public OpenCircuitException(string message, Exception cause, TimeSpan remainingDuration) : base(message, cause) { + RemainingDuration = remainingDuration; } #if SERIALIZATION