Update health-check logic with adaptive thresholds
Replaced the fixed 3x interval multiplier with an adaptive threshold based on the configured interval duration. Health states (`Healthy`, `Degraded`, `Unhealthy`) were redefined to reflect this change. - Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals (≥10s), ensuring faster problem detection for longer intervals and tolerance for jitter in shorter intervals. - Updated `CheckHealthAsync` to calculate `maxAllowedDelay` using the adaptive multiplier. - Added `Multiplier` and `MaxAllowedDelay` to diagnostic data for better monitoring and debugging. - Improved documentation and comments to explain the new logic.
This commit is contained in:
@@ -18,12 +18,14 @@ namespace ECMJobRunner.WebCron.ProfileWorker;
|
||||
/// <item><description>Tracks health status based on success/failure patterns</description></item>
|
||||
/// <item><description>Gracefully handles cancellation during application shutdown</description></item>
|
||||
/// </list>
|
||||
/// Health states:
|
||||
/// Health states (with adaptive thresholds):
|
||||
/// <list type="bullet">
|
||||
/// <item><description><b>Healthy</b>: Last successful run within 3x interval</description></item>
|
||||
/// <item><description><b>Degraded</b>: 1-2 consecutive failures but within time limit</description></item>
|
||||
/// <item><description><b>Unhealthy</b>: No success for 3x interval or 3+ consecutive failures</description></item>
|
||||
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
|
||||
/// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item>
|
||||
/// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)</description></item>
|
||||
/// </list>
|
||||
/// The adaptive multiplier ensures fast problem detection when using longer intervals (e.g., 60s interval = 90s timeout)
|
||||
/// while maintaining tolerance for network jitter on short intervals (e.g., 1s interval = 3s timeout).
|
||||
/// </remarks>
|
||||
public class ProfileWorker(
|
||||
ILogger<ProfileWorker> Logger,
|
||||
@@ -112,20 +114,24 @@ public class ProfileWorker(
|
||||
/// <remarks>
|
||||
/// Health determination logic:
|
||||
/// <list type="number">
|
||||
/// <item><description><b>Unhealthy</b>: Time since last success exceeds 3x the configured interval</description></item>
|
||||
/// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)</description></item>
|
||||
/// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
|
||||
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
|
||||
/// </list>
|
||||
/// Includes diagnostic data in the result for monitoring and alerting.
|
||||
/// Adaptive multiplier ensures fast problem detection for longer intervals while maintaining tolerance for short intervals.
|
||||
/// </remarks>
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
|
||||
var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3);
|
||||
|
||||
// Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
|
||||
// This ensures faster problem detection when using longer intervals (e.g., 60s)
|
||||
var multiplier = _options.IntervalMS < 10000 ? 3.0 : 1.5;
|
||||
var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * multiplier);
|
||||
|
||||
// Unhealthy: No successful run for 3x interval
|
||||
// Unhealthy: No successful run within adaptive threshold
|
||||
if (timeSinceLastSuccess > maxAllowedDelay)
|
||||
{
|
||||
var data = new Dictionary<string, object>
|
||||
@@ -133,7 +139,9 @@ public class ProfileWorker(
|
||||
["LastSuccessfulRun"] = _lastSuccessfulRun,
|
||||
["TimeSinceLastSuccess"] = timeSinceLastSuccess,
|
||||
["ConsecutiveFailures"] = _consecutiveFailures,
|
||||
["IntervalMS"] = _options.IntervalMS
|
||||
["IntervalMS"] = _options.IntervalMS,
|
||||
["Multiplier"] = multiplier,
|
||||
["MaxAllowedDelay"] = maxAllowedDelay
|
||||
};
|
||||
|
||||
if (_lastException != null)
|
||||
|
||||
Reference in New Issue
Block a user