Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Calculated break duration for Circuit breaker #1776

Merged
merged 40 commits into from
Nov 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
9d45467
Add BreakDurationGenerator
atawLee Oct 22, 2023
4880911
Add Duration Generator UnitTest
atawLee Oct 22, 2023
5752bb6
Apply suggestions from code review
atawLee Oct 23, 2023
9e2e5f2
Merge branch 'main' into DurationGenerator
jognhoonlee Oct 26, 2023
22e2590
Add BreakDurationGeneratorArguments
atawLee Oct 26, 2023
29942be
Update circuit-breaker.md
atawLee Oct 26, 2023
030b234
Update circuit-breaker.md
atawLee Oct 26, 2023
b924728
remove duplicate
atawLee Oct 26, 2023
8f9a594
Add BreakDurationGenerator
atawLee Oct 22, 2023
b9023aa
Add Duration Generator UnitTest
atawLee Oct 22, 2023
31ff325
Apply suggestions from code review
atawLee Oct 23, 2023
2faadbd
Add BreakDurationGeneratorArguments
atawLee Oct 26, 2023
491c99f
Update document circuit-breaker.md
atawLee Oct 26, 2023
f071d23
Merge branch 'DurationGenerator' of https://github.com/atawLee/Polly …
atawLee Oct 26, 2023
9d2ae69
fix document
atawLee Oct 26, 2023
b6a8948
Update Document
atawLee Oct 27, 2023
a1c8474
Update CircuitBreaker Document
atawLee Oct 27, 2023
92ec473
Merge branch 'App-vNext:main' into DurationGenerator
atawLee Oct 27, 2023
e24ab51
Apply suggestions from code review
atawLee Oct 29, 2023
39c4721
Apply suggestions from code review
atawLee Oct 29, 2023
5f7095d
Merge branch 'App-vNext:main' into DurationGenerator
atawLee Oct 30, 2023
51620d7
Merge branch 'App-vNext:main' into DurationGenerator
atawLee Oct 30, 2023
92de086
update document
atawLee Oct 30, 2023
3685e9d
Merge branch 'DurationGenerator' of https://github.com/atawLee/Polly …
atawLee Oct 30, 2023
8e5f803
controller - Add context
atawLee Oct 30, 2023
8a82bf0
update unittest
atawLee Oct 30, 2023
cf02659
Apply suggestions from code review
atawLee Nov 2, 2023
f55b6aa
Merge branch 'App-vNext:main' into DurationGenerator
atawLee Nov 2, 2023
efcb36b
Reflect 'ValueTask' requirements
atawLee Nov 2, 2023
fc80c3e
Fixed warning
atawLee Nov 2, 2023
5b1a882
Fixed Warning
atawLee Nov 2, 2023
6962659
update PublishAPI
atawLee Nov 2, 2023
5355b3e
Fixed Unittest
atawLee Nov 2, 2023
5448c32
Delete Korean Comment
atawLee Nov 2, 2023
1e32edd
Pull MainBranch
atawLee Nov 6, 2023
e4c3c7b
Pull Main Branch
atawLee Nov 6, 2023
7044ed4
Merge Main Branch
atawLee Nov 6, 2023
8867436
Update Test Coverage
atawLee Nov 7, 2023
fb81ee6
Apply suggestions from code review
atawLee Nov 7, 2023
01b35fc
#pragma warning disable S1226
atawLee Nov 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
namespace Polly.CircuitBreaker;

#pragma warning disable CA1815 // Override equals and operator equals on value types

/// <summary>
/// Represents arguments used to generate a dynamic break duration for a circuit breaker.
/// </summary>
public readonly struct BreakDurationGeneratorArguments
{
/// <summary>
/// Initializes a new instance of the <see cref="BreakDurationGeneratorArguments"/> struct.
/// </summary>
/// <param name="failureRate">The failure rate at which the circuit breaker should trip.
/// It represents the ratio of failed actions to the total executed actions.</param>
/// <param name="failureCount">The number of failures that have occurred.
/// This count is used to determine if the failure threshold has been reached.</param>
/// <param name="context">The resilience context providing additional information
/// about the execution state and failures.</param>
public BreakDurationGeneratorArguments(
double failureRate,
int failureCount,
ResilienceContext context)
{
FailureRate = failureRate;
FailureCount = failureCount;
Context = context;
}

/// <summary>
/// Gets the failure rate that represents the ratio of failures to total actions.
/// </summary>
public double FailureRate { get; }

/// <summary>
/// Gets the count of failures that have occurred.
/// </summary>
public int FailureCount { get; }

/// <summary>
/// Gets the context that provides additional information about the resilience operation.
/// </summary>
public ResilienceContext Context { get; }
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using System;
using System.ComponentModel.DataAnnotations;
using System.Diagnostics.CodeAnalysis;

Expand Down Expand Up @@ -68,6 +69,14 @@ public class CircuitBreakerStrategyOptions<TResult> : ResilienceStrategyOptions
[UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Addressed with DynamicDependency on ValidationHelper.Validate method")]
public TimeSpan BreakDuration { get; set; } = CircuitBreakerConstants.DefaultBreakDuration;

/// <summary>
/// Gets or sets an optional delegate to use to dynamically generate the break duration.
/// </summary>
/// <value>
/// The default value is <see langword="null"/>.
/// </value>
public Func<BreakDurationGeneratorArguments, ValueTask<TimeSpan>>? BreakDurationGenerator { get; set; }

/// <summary>
/// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker.
/// </summary>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,5 +44,7 @@ public override void OnActionFailure(CircuitState currentState, out bool shouldB
}

public override void OnCircuitClosed() => _metrics.Reset();
public override int FailureCount => _metrics.GetHealthInfo().FailureCount;
public override double FailureRate => _metrics.GetHealthInfo().FailureRate;
}

2 changes: 2 additions & 0 deletions src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,6 @@ internal abstract class CircuitBehavior
public abstract void OnActionFailure(CircuitState currentState, out bool shouldBreak);

public abstract void OnCircuitClosed();
public abstract int FailureCount { get; }
public abstract double FailureRate { get; }
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,24 @@ internal sealed class CircuitStateController<T> : IDisposable
private readonly ResilienceStrategyTelemetry _telemetry;
private readonly CircuitBehavior _behavior;
private readonly TimeSpan _breakDuration;
private readonly Func<BreakDurationGeneratorArguments, ValueTask<TimeSpan>>? _breakDurationGenerator;
private DateTimeOffset _blockedUntil;
private CircuitState _circuitState = CircuitState.Closed;
private Outcome<T>? _lastOutcome;
private BrokenCircuitException _breakingException = new();
private bool _disposed;

#pragma warning disable S107
public CircuitStateController(
TimeSpan breakDuration,
Func<OnCircuitOpenedArguments<T>, ValueTask>? onOpened,
Func<OnCircuitClosedArguments<T>, ValueTask>? onClosed,
Func<OnCircuitHalfOpenedArguments, ValueTask>? onHalfOpen,
CircuitBehavior behavior,
TimeProvider timeProvider,
ResilienceStrategyTelemetry telemetry)
ResilienceStrategyTelemetry telemetry,
Func<BreakDurationGeneratorArguments, ValueTask<TimeSpan>>? breakDurationGenerator = null)
#pragma warning restore S107
{
_breakDuration = breakDuration;
_onOpened = onOpened;
Expand All @@ -38,6 +42,7 @@ public CircuitStateController(
_behavior = behavior;
_timeProvider = timeProvider;
_telemetry = telemetry;
_breakDurationGenerator = breakDurationGenerator;
}

public CircuitState CircuitState
Expand Down Expand Up @@ -314,6 +319,15 @@ private void OpenCircuitFor_NeedsLock(Outcome<T> outcome, TimeSpan breakDuration
scheduledTask = null;
var utcNow = _timeProvider.GetUtcNow();

if (_breakDurationGenerator is not null)
{
#pragma warning disable CA2012
#pragma warning disable S1226
martincostello marked this conversation as resolved.
Show resolved Hide resolved
breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)).GetAwaiter().GetResult();
#pragma warning restore S1226
#pragma warning restore CA2012
}

_blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration;

var transitionedState = _circuitState;
Expand Down
6 changes: 3 additions & 3 deletions src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
namespace Polly.CircuitBreaker.Health;

internal readonly record struct HealthInfo(int Throughput, double FailureRate)
internal readonly record struct HealthInfo(int Throughput, double FailureRate, int FailureCount)
{
public static HealthInfo Create(int successes, int failures)
{
var total = successes + failures;
if (total == 0)
{
return new HealthInfo(0, 0);
return new HealthInfo(0, 0, failures);
}

return new(total, failures / (double)total);
return new(total, failures / (double)total, failures);
}
}
8 changes: 8 additions & 0 deletions src/Polly.Core/PublicAPI.Unshipped.txt
Original file line number Diff line number Diff line change
@@ -1 +1,9 @@
#nullable enable
Polly.CircuitBreaker.BreakDurationGeneratorArguments
Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments() -> void
Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments(double failureRate, int failureCount, Polly.ResilienceContext! context) -> void
Polly.CircuitBreaker.BreakDurationGeneratorArguments.Context.get -> Polly.ResilienceContext!
Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureCount.get -> int
Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureRate.get -> double
Polly.CircuitBreaker.CircuitBreakerStrategyOptions<TResult>.BreakDurationGenerator.get -> System.Func<Polly.CircuitBreaker.BreakDurationGeneratorArguments, System.Threading.Tasks.ValueTask<System.TimeSpan>>?
Polly.CircuitBreaker.CircuitBreakerStrategyOptions<TResult>.BreakDurationGenerator.set -> void
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
using Polly;
using Polly.CircuitBreaker;

namespace Polly.Core.Tests.CircuitBreaker;
atawLee marked this conversation as resolved.
Show resolved Hide resolved

public class BreakDurationGeneratorArgumentsTests
{
[Fact]
public void Constructor_ShouldSetFailureRate()
{
double expectedFailureRate = 0.5;
int failureCount = 10;
var context = new ResilienceContext();

var args = new BreakDurationGeneratorArguments(expectedFailureRate, failureCount, context);

args.FailureRate.Should().Be(expectedFailureRate);
}

[Fact]
public void Constructor_ShouldSetFailureCount()
{
double failureRate = 0.5;
int expectedFailureCount = 10;
var context = new ResilienceContext();

var args = new BreakDurationGeneratorArguments(failureRate, expectedFailureCount, context);

args.FailureCount.Should().Be(expectedFailureCount);
}

[Fact]
public void Constructor_ShouldSetContext()
{
double failureRate = 0.5;
int failureCount = 10;
var expectedContext = new ResilienceContext();

var args = new BreakDurationGeneratorArguments(failureRate, failureCount, expectedContext);

args.Context.Should().Be(expectedContext);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,21 @@ public class AdvancedCircuitBehaviorTests
{
private HealthMetrics _metrics = Substitute.For<HealthMetrics>(TimeProvider.System);

[InlineData(10, 10, 0.0, 0.1, false)]
[InlineData(10, 10, 0.1, 0.1, true)]
[InlineData(10, 10, 0.2, 0.1, true)]
[InlineData(11, 10, 0.2, 0.1, true)]
[InlineData(9, 10, 0.1, 0.1, false)]
[InlineData(10, 10, 0.0, 0.1, 0, false)]
[InlineData(10, 10, 0.1, 0.1, 1, true)]
[InlineData(10, 10, 0.2, 0.1, 2, true)]
[InlineData(11, 10, 0.2, 0.1, 3, true)]
[InlineData(9, 10, 0.1, 0.1, 4, false)]
[Theory]
public void OnActionFailure_WhenClosed_EnsureCorrectBehavior(
int throughput,
int minimumThruput,
double failureRate,
double failureThreshold,
int failureCount,
bool expectedShouldBreak)
{
_metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate));
_metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount));

var behavior = new AdvancedCircuitBehavior(failureThreshold, minimumThruput, _metrics);

Expand All @@ -41,7 +42,6 @@ public void OnActionFailure_State_EnsureCorrectCalls(CircuitState state, bool sh
_metrics = Substitute.For<HealthMetrics>(TimeProvider.System);

var sut = Create();

sut.OnActionFailure(state, out var shouldBreak);

shouldBreak.Should().BeFalse();
Expand All @@ -66,6 +66,27 @@ public void OnCircuitClosed_Ok()
_metrics.Received(1).Reset();
}

[Theory]
[InlineData(10, 0.0, 0)]
[InlineData(10, 0.1, 1)]
[InlineData(10, 0.2, 2)]
[InlineData(11, 0.2, 3)]
[InlineData(9, 0.1, 4)]
public void BehaviorProperties_ShouldReflectHealthInfoValues(
int throughput,
double failureRate,
int failureCount)
{
var anyFailureThreshold = 10;
var anyMinimumThruput = 100;

_metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount));
var behavior = new AdvancedCircuitBehavior(anyFailureThreshold, anyMinimumThruput, _metrics);

behavior.FailureCount.Should().Be(failureCount, "because the FailureCount should match the HealthInfo");
behavior.FailureRate.Should().Be(failureRate, "because the FailureRate should match the HealthInfo");
}

atawLee marked this conversation as resolved.
Show resolved Hide resolved
private AdvancedCircuitBehavior Create()
{
return new(CircuitBreakerConstants.DefaultFailureRatio, CircuitBreakerConstants.DefaultMinimumThroughput, _metrics);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,41 @@ public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state,
}
}

[Fact]
public async Task OnActionFailureAsync_EnsureBreakDurationGeneration()
{
// arrange
using var controller = CreateController(new()
{
FailureRatio = 0,
MinimumThroughput = 0,
SamplingDuration = default,
BreakDuration = TimeSpan.FromMinutes(1),
BreakDurationGenerator = static args => new ValueTask<TimeSpan>(TimeSpan.FromMinutes(args.FailureCount)),
OnClosed = null,
OnOpened = null,
OnHalfOpened = null,
ManualControl = null,
StateProvider = null
});

await TransitionToState(controller, CircuitState.Closed);

var utcNow = DateTimeOffset.MaxValue;

_timeProvider.SetUtcNow(utcNow);
_circuitBehavior.FailureCount.Returns(1);
_circuitBehavior.When(v => v.OnActionFailure(CircuitState.Closed, out Arg.Any<bool>()))
.Do(x => x[1] = true);

// act
await controller.OnActionFailureAsync(Outcome.FromResult(99), ResilienceContextPool.Shared.Get());

// assert
var blockedTill = GetBlockedTill(controller);
blockedTill.Should().Be(utcNow);
}

[InlineData(true)]
[InlineData(false)]
[Theory]
Expand Down Expand Up @@ -470,4 +505,14 @@ private async Task OpenCircuit(CircuitStateController<int> controller, Outcome<i
_circuitBehavior,
_timeProvider,
TestUtilities.CreateResilienceTelemetry(_telemetryListener));

private CircuitStateController<int> CreateController(CircuitBreakerStrategyOptions<int> options) => new(
options.BreakDuration,
options.OnOpened,
options.OnClosed,
options.OnHalfOpened,
_circuitBehavior,
_timeProvider,
TestUtilities.CreateResilienceTelemetry(_telemetryListener),
options.BreakDurationGenerator);
}
46 changes: 46 additions & 0 deletions test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,50 @@ public void Create_Ok(int samplingDurationMs, Type expectedType)
.Should()
.BeOfType(expectedType);
}

[Fact]
public void HealthInfo_WithZeroTotal_ShouldSetValuesCorrectly()
{
// Arrange & Act
var result = HealthInfo.Create(0, 0);

// Assert
result.Throughput.Should().Be(0);
result.FailureRate.Should().Be(0);
result.FailureCount.Should().Be(0);
}

[Fact]
public void HealthInfo_ParameterizedConstructor_ShouldSetProperties()
{
// Arrange
int expectedThroughput = 100;
double expectedFailureRate = 0.25;
int expectedFailureCount = 25;

// Act
var result = new HealthInfo(expectedThroughput, expectedFailureRate, expectedFailureCount);

// Assert
result.Throughput.Should().Be(expectedThroughput);
result.FailureRate.Should().Be(expectedFailureRate);
result.FailureCount.Should().Be(expectedFailureCount);
}

[Fact]
public void HealthInfo_Constructor_ShouldSetValuesCorrectly()
{
// Arrange
int throughput = 10;
double failureRate = 0.2;
int failureCount = 2;

// Act
var result = new HealthInfo(throughput, failureRate, failureCount);

// Assert
result.Throughput.Should().Be(throughput);
result.FailureRate.Should().Be(failureRate);
result.FailureCount.Should().Be(failureCount);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,11 @@ public void GetHealthInfo_EnsureWindowRespected()
_timeProvider.Advance(TimeSpan.FromSeconds(2));
health.Add(metrics.GetHealthInfo());

health[0].Should().Be(new HealthInfo(2, 0.5));
health[1].Should().Be(new HealthInfo(4, 0.5));
health[3].Should().Be(new HealthInfo(8, 0.25));
health[4].Should().Be(new HealthInfo(8, 0.125));
health[5].Should().Be(new HealthInfo(6, 0.0));
health[0].Should().Be(new HealthInfo(2, 0.5, 1));
health[1].Should().Be(new HealthInfo(4, 0.5, 2));
health[3].Should().Be(new HealthInfo(8, 0.25, 2));
health[4].Should().Be(new HealthInfo(8, 0.125, 1));
health[5].Should().Be(new HealthInfo(6, 0.0, 0));
}

[Fact]
Expand Down Expand Up @@ -109,7 +109,7 @@ public void GetHealthInfo_SamplingDurationRespected(bool variance)

_timeProvider.Advance(_samplingDuration + (variance ? TimeSpan.FromMilliseconds(1) : TimeSpan.Zero));

metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0));
metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0, 0));
}

private RollingHealthMetrics Create() => new(_samplingDuration, _windows, _timeProvider);
Expand Down