Calculated Circuit breaker durations (#1776)

Add support for calculating how long a circuit should break for dynamically.

src/Polly.Core/CircuitBreaker/BreakDurationGeneratorArguments.cs

@@ -0,0 +1,43 @@
+namespace Polly.CircuitBreaker;
+
+#pragma warning disable CA1815 // Override equals and operator equals on value types
+
+/// <summary>
+/// Represents arguments used to generate a dynamic break duration for a circuit breaker.
+/// </summary>
+public readonly struct BreakDurationGeneratorArguments
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="BreakDurationGeneratorArguments"/> struct.
+    /// </summary>
+    /// <param name="failureRate">The failure rate at which the circuit breaker should trip.
+    /// It represents the ratio of failed actions to the total executed actions.</param>
+    /// <param name="failureCount">The number of failures that have occurred.
+    /// This count is used to determine if the failure threshold has been reached.</param>
+    /// <param name="context">The resilience context providing additional information
+    /// about the execution state and failures.</param>
+    public BreakDurationGeneratorArguments(
+        double failureRate,
+        int failureCount,
+        ResilienceContext context)
+    {
+        FailureRate = failureRate;
+        FailureCount = failureCount;
+        Context = context;
+    }
+
+    /// <summary>
+    /// Gets the failure rate that represents the ratio of failures to total actions.
+    /// </summary>
+    public double FailureRate { get; }
+
+    /// <summary>
+    /// Gets the count of failures that have occurred.
+    /// </summary>
+    public int FailureCount { get; }
+
+    /// <summary>
+    /// Gets the context that provides additional information about the resilience operation.
+    /// </summary>
+    public ResilienceContext Context { get; }
+}

src/Polly.Core/CircuitBreaker/CircuitBreakerStrategyOptions.TResult.cs

@@ -1,3 +1,4 @@
+using System;
 using System.ComponentModel.DataAnnotations;
 using System.Diagnostics.CodeAnalysis;
 
@@ -68,6 +69,14 @@ public class CircuitBreakerStrategyOptions<TResult> : ResilienceStrategyOptions
     [UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Addressed with DynamicDependency on ValidationHelper.Validate method")]
     public TimeSpan BreakDuration { get; set; } = CircuitBreakerConstants.DefaultBreakDuration;
 
+    /// <summary>
+    /// Gets or sets an optional delegate to use to dynamically generate the break duration.
+    /// </summary>
+    /// <value>
+    /// The default value is <see langword="null"/>.
+    /// </value>
+    public Func<BreakDurationGeneratorArguments, ValueTask<TimeSpan>>? BreakDurationGenerator { get; set; }
+
     /// <summary>
     /// Gets or sets a predicate that determines whether the outcome should be handled by the circuit breaker.
     /// </summary>

src/Polly.Core/CircuitBreaker/Controller/AdvancedCircuitBehavior.cs

@@ -44,5 +44,7 @@ public override void OnActionFailure(CircuitState currentState, out bool shouldB
     }
 
     public override void OnCircuitClosed() => _metrics.Reset();
+    public override int FailureCount => _metrics.GetHealthInfo().FailureCount;
+    public override double FailureRate => _metrics.GetHealthInfo().FailureRate;
 }
 

src/Polly.Core/CircuitBreaker/Controller/CircuitBehavior.cs

@@ -10,4 +10,6 @@ internal abstract class CircuitBehavior
     public abstract void OnActionFailure(CircuitState currentState, out bool shouldBreak);
 
     public abstract void OnCircuitClosed();
+    public abstract int FailureCount { get; }
+    public abstract double FailureRate { get; }
 }

src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs

@@ -16,20 +16,24 @@ internal sealed class CircuitStateController<T> : IDisposable
     private readonly ResilienceStrategyTelemetry _telemetry;
     private readonly CircuitBehavior _behavior;
     private readonly TimeSpan _breakDuration;
+    private readonly Func<BreakDurationGeneratorArguments, ValueTask<TimeSpan>>? _breakDurationGenerator;
     private DateTimeOffset _blockedUntil;
     private CircuitState _circuitState = CircuitState.Closed;
     private Outcome<T>? _lastOutcome;
     private BrokenCircuitException _breakingException = new();
     private bool _disposed;
 
+#pragma warning disable S107
     public CircuitStateController(
         TimeSpan breakDuration,
         Func<OnCircuitOpenedArguments<T>, ValueTask>? onOpened,
         Func<OnCircuitClosedArguments<T>, ValueTask>? onClosed,
         Func<OnCircuitHalfOpenedArguments, ValueTask>? onHalfOpen,
         CircuitBehavior behavior,
         TimeProvider timeProvider,
-        ResilienceStrategyTelemetry telemetry)
+        ResilienceStrategyTelemetry telemetry,
+        Func<BreakDurationGeneratorArguments, ValueTask<TimeSpan>>? breakDurationGenerator = null)
+#pragma warning restore S107
     {
         _breakDuration = breakDuration;
         _onOpened = onOpened;
@@ -38,6 +42,7 @@ public CircuitStateController(
         _behavior = behavior;
         _timeProvider = timeProvider;
         _telemetry = telemetry;
+        _breakDurationGenerator = breakDurationGenerator;
     }
 
     public CircuitState CircuitState
@@ -314,6 +319,15 @@ private void OpenCircuitFor_NeedsLock(Outcome<T> outcome, TimeSpan breakDuration
         scheduledTask = null;
         var utcNow = _timeProvider.GetUtcNow();
 
+        if (_breakDurationGenerator is not null)
+        {
+#pragma warning disable CA2012
+#pragma warning disable S1226
+            breakDuration = _breakDurationGenerator(new(_behavior.FailureRate, _behavior.FailureCount, context)).GetAwaiter().GetResult();
+#pragma warning restore S1226
+#pragma warning restore CA2012
+        }
+
         _blockedUntil = IsDateTimeOverflow(utcNow, breakDuration) ? DateTimeOffset.MaxValue : utcNow + breakDuration;
 
         var transitionedState = _circuitState;

src/Polly.Core/CircuitBreaker/Health/HealthInfo.cs

@@ -1,15 +1,15 @@
 namespace Polly.CircuitBreaker.Health;
 
-internal readonly record struct HealthInfo(int Throughput, double FailureRate)
+internal readonly record struct HealthInfo(int Throughput, double FailureRate, int FailureCount)
 {
     public static HealthInfo Create(int successes, int failures)
     {
         var total = successes + failures;
         if (total == 0)
         {
-            return new HealthInfo(0, 0);
+            return new HealthInfo(0, 0, failures);
         }
 
-        return new(total, failures / (double)total);
+        return new(total, failures / (double)total, failures);
     }
 }

src/Polly.Core/PublicAPI.Unshipped.txt

@@ -1 +1,9 @@
 #nullable enable
+Polly.CircuitBreaker.BreakDurationGeneratorArguments
+Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments() -> void
+Polly.CircuitBreaker.BreakDurationGeneratorArguments.BreakDurationGeneratorArguments(double failureRate, int failureCount, Polly.ResilienceContext! context) -> void
+Polly.CircuitBreaker.BreakDurationGeneratorArguments.Context.get -> Polly.ResilienceContext!
+Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureCount.get -> int
+Polly.CircuitBreaker.BreakDurationGeneratorArguments.FailureRate.get -> double
+Polly.CircuitBreaker.CircuitBreakerStrategyOptions<TResult>.BreakDurationGenerator.get -> System.Func<Polly.CircuitBreaker.BreakDurationGeneratorArguments, System.Threading.Tasks.ValueTask<System.TimeSpan>>?
+Polly.CircuitBreaker.CircuitBreakerStrategyOptions<TResult>.BreakDurationGenerator.set -> void

test/Polly.Core.Tests/CircuitBreaker/BreakDurationGeneratorArgumentsTests.cs

@@ -0,0 +1,43 @@
+using Polly;
+using Polly.CircuitBreaker;
+
+namespace Polly.Core.Tests.CircuitBreaker;
+
+public class BreakDurationGeneratorArgumentsTests
+{
+    [Fact]
+    public void Constructor_ShouldSetFailureRate()
+    {
+        double expectedFailureRate = 0.5;
+        int failureCount = 10;
+        var context = new ResilienceContext();
+
+        var args = new BreakDurationGeneratorArguments(expectedFailureRate, failureCount, context);
+
+        args.FailureRate.Should().Be(expectedFailureRate);
+    }
+
+    [Fact]
+    public void Constructor_ShouldSetFailureCount()
+    {
+        double failureRate = 0.5;
+        int expectedFailureCount = 10;
+        var context = new ResilienceContext();
+
+        var args = new BreakDurationGeneratorArguments(failureRate, expectedFailureCount, context);
+
+        args.FailureCount.Should().Be(expectedFailureCount);
+    }
+
+    [Fact]
+    public void Constructor_ShouldSetContext()
+    {
+        double failureRate = 0.5;
+        int failureCount = 10;
+        var expectedContext = new ResilienceContext();
+
+        var args = new BreakDurationGeneratorArguments(failureRate, failureCount, expectedContext);
+
+        args.Context.Should().Be(expectedContext);
+    }
+}

test/Polly.Core.Tests/CircuitBreaker/Controller/AdvancedCircuitBehaviorTests.cs

@@ -8,20 +8,21 @@ public class AdvancedCircuitBehaviorTests
 {
     private HealthMetrics _metrics = Substitute.For<HealthMetrics>(TimeProvider.System);
 
-    [InlineData(10, 10, 0.0, 0.1, false)]
-    [InlineData(10, 10, 0.1, 0.1, true)]
-    [InlineData(10, 10, 0.2, 0.1, true)]
-    [InlineData(11, 10, 0.2, 0.1, true)]
-    [InlineData(9, 10, 0.1, 0.1, false)]
+    [InlineData(10, 10, 0.0, 0.1, 0, false)]
+    [InlineData(10, 10, 0.1, 0.1, 1, true)]
+    [InlineData(10, 10, 0.2, 0.1, 2, true)]
+    [InlineData(11, 10, 0.2, 0.1, 3, true)]
+    [InlineData(9, 10, 0.1, 0.1, 4, false)]
     [Theory]
     public void OnActionFailure_WhenClosed_EnsureCorrectBehavior(
         int throughput,
         int minimumThruput,
         double failureRate,
         double failureThreshold,
+        int failureCount,
         bool expectedShouldBreak)
     {
-        _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate));
+        _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount));
 
         var behavior = new AdvancedCircuitBehavior(failureThreshold, minimumThruput, _metrics);
 
@@ -41,7 +42,6 @@ public void OnActionFailure_State_EnsureCorrectCalls(CircuitState state, bool sh
         _metrics = Substitute.For<HealthMetrics>(TimeProvider.System);
 
         var sut = Create();
-
         sut.OnActionFailure(state, out var shouldBreak);
 
         shouldBreak.Should().BeFalse();
@@ -66,6 +66,27 @@ public void OnCircuitClosed_Ok()
         _metrics.Received(1).Reset();
     }
 
+    [Theory]
+    [InlineData(10, 0.0, 0)]
+    [InlineData(10, 0.1, 1)]
+    [InlineData(10, 0.2, 2)]
+    [InlineData(11, 0.2, 3)]
+    [InlineData(9, 0.1, 4)]
+    public void BehaviorProperties_ShouldReflectHealthInfoValues(
+        int throughput,
+        double failureRate,
+        int failureCount)
+    {
+        var anyFailureThreshold = 10;
+        var anyMinimumThruput = 100;
+
+        _metrics.GetHealthInfo().Returns(new HealthInfo(throughput, failureRate, failureCount));
+        var behavior = new AdvancedCircuitBehavior(anyFailureThreshold, anyMinimumThruput, _metrics);
+
+        behavior.FailureCount.Should().Be(failureCount, "because the FailureCount should match the HealthInfo");
+        behavior.FailureRate.Should().Be(failureRate, "because the FailureRate should match the HealthInfo");
+    }
+
     private AdvancedCircuitBehavior Create()
     {
         return new(CircuitBreakerConstants.DefaultFailureRatio, CircuitBreakerConstants.DefaultMinimumThroughput, _metrics);

test/Polly.Core.Tests/CircuitBreaker/Controller/CircuitStateControllerTests.cs

@@ -305,6 +305,41 @@ public async Task OnActionFailureAsync_EnsureCorrectBehavior(CircuitState state,
         }
     }
 
+    [Fact]
+    public async Task OnActionFailureAsync_EnsureBreakDurationGeneration()
+    {
+        // arrange
+        using var controller = CreateController(new()
+        {
+            FailureRatio = 0,
+            MinimumThroughput = 0,
+            SamplingDuration = default,
+            BreakDuration = TimeSpan.FromMinutes(1),
+            BreakDurationGenerator = static args => new ValueTask<TimeSpan>(TimeSpan.FromMinutes(args.FailureCount)),
+            OnClosed = null,
+            OnOpened = null,
+            OnHalfOpened = null,
+            ManualControl = null,
+            StateProvider = null
+        });
+
+        await TransitionToState(controller, CircuitState.Closed);
+
+        var utcNow = DateTimeOffset.MaxValue;
+
+        _timeProvider.SetUtcNow(utcNow);
+        _circuitBehavior.FailureCount.Returns(1);
+        _circuitBehavior.When(v => v.OnActionFailure(CircuitState.Closed, out Arg.Any<bool>()))
+            .Do(x => x[1] = true);
+
+        // act
+        await controller.OnActionFailureAsync(Outcome.FromResult(99), ResilienceContextPool.Shared.Get());
+
+        // assert
+        var blockedTill = GetBlockedTill(controller);
+        blockedTill.Should().Be(utcNow);
+    }
+
     [InlineData(true)]
     [InlineData(false)]
     [Theory]
@@ -470,4 +505,14 @@ private async Task OpenCircuit(CircuitStateController<int> controller, Outcome<i
         _circuitBehavior,
         _timeProvider,
         TestUtilities.CreateResilienceTelemetry(_telemetryListener));
+
+    private CircuitStateController<int> CreateController(CircuitBreakerStrategyOptions<int> options) => new(
+        options.BreakDuration,
+        options.OnOpened,
+        options.OnClosed,
+        options.OnHalfOpened,
+        _circuitBehavior,
+        _timeProvider,
+        TestUtilities.CreateResilienceTelemetry(_telemetryListener),
+        options.BreakDurationGenerator);
 }

test/Polly.Core.Tests/CircuitBreaker/Health/HealthMetricsTests.cs

@@ -17,4 +17,50 @@ public void Create_Ok(int samplingDurationMs, Type expectedType)
             .Should()
             .BeOfType(expectedType);
     }
+
+    [Fact]
+    public void HealthInfo_WithZeroTotal_ShouldSetValuesCorrectly()
+    {
+        // Arrange & Act
+        var result = HealthInfo.Create(0, 0);
+
+        // Assert
+        result.Throughput.Should().Be(0);
+        result.FailureRate.Should().Be(0);
+        result.FailureCount.Should().Be(0);
+    }
+
+    [Fact]
+    public void HealthInfo_ParameterizedConstructor_ShouldSetProperties()
+    {
+        // Arrange
+        int expectedThroughput = 100;
+        double expectedFailureRate = 0.25;
+        int expectedFailureCount = 25;
+
+        // Act
+        var result = new HealthInfo(expectedThroughput, expectedFailureRate, expectedFailureCount);
+
+        // Assert
+        result.Throughput.Should().Be(expectedThroughput);
+        result.FailureRate.Should().Be(expectedFailureRate);
+        result.FailureCount.Should().Be(expectedFailureCount);
+    }
+
+    [Fact]
+    public void HealthInfo_Constructor_ShouldSetValuesCorrectly()
+    {
+        // Arrange
+        int throughput = 10;
+        double failureRate = 0.2;
+        int failureCount = 2;
+
+        // Act
+        var result = new HealthInfo(throughput, failureRate, failureCount);
+
+        // Assert
+        result.Throughput.Should().Be(throughput);
+        result.FailureRate.Should().Be(failureRate);
+        result.FailureCount.Should().Be(failureCount);
+    }
 }

test/Polly.Core.Tests/CircuitBreaker/Health/RollingHealthMetricsTests.cs

@@ -62,11 +62,11 @@ public void GetHealthInfo_EnsureWindowRespected()
         _timeProvider.Advance(TimeSpan.FromSeconds(2));
         health.Add(metrics.GetHealthInfo());
 
-        health[0].Should().Be(new HealthInfo(2, 0.5));
-        health[1].Should().Be(new HealthInfo(4, 0.5));
-        health[3].Should().Be(new HealthInfo(8, 0.25));
-        health[4].Should().Be(new HealthInfo(8, 0.125));
-        health[5].Should().Be(new HealthInfo(6, 0.0));
+        health[0].Should().Be(new HealthInfo(2, 0.5, 1));
+        health[1].Should().Be(new HealthInfo(4, 0.5, 2));
+        health[3].Should().Be(new HealthInfo(8, 0.25, 2));
+        health[4].Should().Be(new HealthInfo(8, 0.125, 1));
+        health[5].Should().Be(new HealthInfo(6, 0.0, 0));
     }
 
     [Fact]
@@ -109,7 +109,7 @@ public void GetHealthInfo_SamplingDurationRespected(bool variance)
 
         _timeProvider.Advance(_samplingDuration + (variance ? TimeSpan.FromMilliseconds(1) : TimeSpan.Zero));
 
-        metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0));
+        metrics.GetHealthInfo().Should().Be(new HealthInfo(0, 0, 0));
     }
 
     private RollingHealthMetrics Create() => new(_samplingDuration, _windows, _timeProvider);