Skip to content

Commit

Permalink
Added fixed buckets based on duration provided. Added single spec to …
Browse files Browse the repository at this point in the history
…ensure it worked. Still missing several more to make certain that it works as expected.
  • Loading branch information
kristianhald committed Apr 6, 2016
1 parent 836e1d6 commit 0b0bb45
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 24 deletions.
111 changes: 87 additions & 24 deletions src/Polly.Shared/CircuitBreaker/TimesliceCircuitController.cs
Original file line number Diff line number Diff line change
@@ -1,49 +1,113 @@
using System;
using System.Collections.Generic;
using Polly.Utilities;

namespace Polly.CircuitBreaker
{
internal class TimesliceCircuitController : CircuitStateController
{
private readonly long _timesliceDuration;
private readonly HealthMetrics _metrics;
private readonly double _failureThreshold;
private readonly int _minimumThroughput;

private HealthMetric _metric;

private class HealthMetric // If only one metric at a time is ever retained, this could be removed (for performance) and the properties incorporated in to the parent class.
private class HealthMetricSlice // If only one metric at a time is ever retained, this could be removed (for performance) and the properties incorporated in to the parent class.
{
public int Successes { get; set; }
public int Failures { get; set; }
public long StartedAt { get; set; }
}

public TimesliceCircuitController(double failureThreshold, TimeSpan timesliceDuration, int minimumThroughput, TimeSpan durationOfBreak, Action<Exception, TimeSpan, Context> onBreak, Action<Context> onReset, Action onHalfOpen) : base(durationOfBreak, onBreak, onReset, onHalfOpen)
private class HealthMetric
{
_timesliceDuration = timesliceDuration.Ticks;
_failureThreshold = failureThreshold;
_minimumThroughput = minimumThroughput;
public int Successes { get; set; }
public int Failures { get; set; }
}

public override void OnCircuitReset(Context context)
private class HealthMetrics
{
using (TimedLock.Lock(_lock))
private readonly long _timeDuration;
private readonly long _timesliceDuration;

private HealthMetricSlice _currentSlice;
private readonly Queue<HealthMetricSlice> _slices = new Queue<HealthMetricSlice>();

public HealthMetrics(TimeSpan timesliceDuration)
{
_metric = null;
_timeDuration = timesliceDuration.Ticks;
_timesliceDuration = _timeDuration / 10; // At the moment it always selects 10 buckets
}

ResetInternal_NeedsLock(context);
public void IncrementSuccess_NeedsLock()
{
// (future enhancement) Any operation in this method disposing of an existing _metric could emit it to a delegate, for health-monitoring capture ...

ActualiseCurrentSlice_NeedsLock();

_currentSlice.Successes++;
}

public void IncrementFailure_NeedsLock()
{
ActualiseCurrentSlice_NeedsLock();

_currentSlice.Failures++;
}

public void Reset_NeedsLock()
{
_currentSlice = null;
_slices.Clear();
}

public HealthMetric GetHealthCounts_NeedsLock()
{
long now = SystemClock.UtcNow().Ticks;
while (now - _slices.Peek().StartedAt >= _timeDuration)
_slices.Dequeue();

int successes = 0;
int failures = 0;
foreach (var slice in _slices)
{
successes += slice.Successes;
failures += slice.Failures;
}

return new HealthMetric
{
Successes = successes,
Failures = failures
};
}

private void ActualiseCurrentSlice_NeedsLock()
{
long now = SystemClock.UtcNow().Ticks;
if (_currentSlice == null || now - _currentSlice.StartedAt >= _timesliceDuration)
{
_currentSlice = new HealthMetricSlice { StartedAt = now };
_slices.Enqueue(_currentSlice);
}
}
}

private void ActualiseCurrentMetric_NeedsLock()
public TimesliceCircuitController(double failureThreshold, TimeSpan timesliceDuration, int minimumThroughput, TimeSpan durationOfBreak, Action<Exception, TimeSpan, Context> onBreak, Action<Context> onReset, Action onHalfOpen) : base(durationOfBreak, onBreak, onReset, onHalfOpen)
{
// (future enhancement) Any operation in this method disposing of an existing _metric could emit it to a delegate, for health-monitoring capture ...

long now = SystemClock.UtcNow().Ticks;
_metrics = new HealthMetrics(timesliceDuration);
_failureThreshold = failureThreshold;
_minimumThroughput = minimumThroughput;
}

if (_metric == null || now - _metric.StartedAt >= _timesliceDuration)
public override void OnCircuitReset(Context context)
{
using (TimedLock.Lock(_lock))
{
_metric = new HealthMetric { StartedAt = now };
// Is only null during initialization of the current class
// as the variable is not set, before the base class calls
// current method from constructor.
_metrics?.Reset_NeedsLock();

ResetInternal_NeedsLock(context);
}
}

Expand All @@ -53,8 +117,7 @@ public override void OnActionSuccess(Context context)
{
if (_circuitState == CircuitState.HalfOpen) { OnCircuitReset(context); }

ActualiseCurrentMetric_NeedsLock();
_metric.Successes++;
_metrics.IncrementSuccess_NeedsLock();
}
}

Expand All @@ -70,11 +133,11 @@ public override void OnActionFailure(Exception ex, Context context)
return;
}

ActualiseCurrentMetric_NeedsLock();
_metric.Failures++;
_metrics.IncrementFailure_NeedsLock();

int throughput = _metric.Failures + _metric.Successes;
if (throughput >= _minimumThroughput && ((double)_metric.Failures) / throughput >= _failureThreshold)
var metric = _metrics.GetHealthCounts_NeedsLock();
int throughput = metric.Failures + metric.Successes;
if (throughput >= _minimumThroughput && ((double)metric.Failures) / throughput >= _failureThreshold)
{
Break_NeedsLock(context);
}
Expand Down
50 changes: 50 additions & 0 deletions src/Polly.SharedSpecs/TimesliceCircuitBreakerSpecs.cs
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,56 @@ public void Should_open_circuit_with_the_last_raised_exception_if_failure_thresh

}

[Fact]
public void Should_open_circuit_with_the_last_raised_exception_if_failure_threshold_equalled_and_throughput_threshold_equalled_even_if_time_is_just_outside_duration()
{
var time = 1.January(2000);
SystemClock.UtcNow = () => time;

var timesliceDuration = TimeSpan.FromSeconds(10);

CircuitBreakerPolicy breaker = Policy
.Handle<DivideByZeroException>()
.TimesliceCircuitBreaker(
failureThreshold: 0.5,
timesliceDuration: timesliceDuration,
minimumThroughput: 4,
durationOfBreak: TimeSpan.FromSeconds(30)
);

// Four of four actions in this test throw handled failures.
breaker.Invoking(x => x.RaiseException<DivideByZeroException>())
.ShouldThrow<DivideByZeroException>();
breaker.CircuitState.Should().Be(CircuitState.Closed);

SystemClock.UtcNow = () => time.AddTicks(timesliceDuration.Ticks / 2);

breaker.Invoking(x => x.RaiseException<DivideByZeroException>())
.ShouldThrow<DivideByZeroException>();
breaker.CircuitState.Should().Be(CircuitState.Closed);

breaker.Invoking(x => x.RaiseException<DivideByZeroException>())
.ShouldThrow<DivideByZeroException>();
breaker.CircuitState.Should().Be(CircuitState.Closed);

// Adjust SystemClock so that timeslice doesn't quite expire; fourth exception thrown in same timeslice.
SystemClock.UtcNow = () => time.AddTicks(timesliceDuration.Ticks + 1);

breaker.Invoking(x => x.RaiseException<DivideByZeroException>())
.ShouldThrow<DivideByZeroException>();
breaker.CircuitState.Should().Be(CircuitState.Closed);

breaker.Invoking(x => x.RaiseException<DivideByZeroException>())
.ShouldThrow<DivideByZeroException>();
breaker.CircuitState.Should().Be(CircuitState.Open);

breaker.Invoking(x => x.RaiseException<DivideByZeroException>())
.ShouldThrow<BrokenCircuitException>()
.WithMessage("The circuit is now open and is not allowing calls.")
.WithInnerException<DivideByZeroException>();
breaker.CircuitState.Should().Be(CircuitState.Open);
}

[Fact]
public void Should_not_open_circuit_if_failure_threshold_not_met_and_throughput_threshold_not_met()
{
Expand Down

0 comments on commit 0b0bb45

Please sign in to comment.