AppStd/ECMJobRunner
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Update health-check logic with adaptive thresholds

Browse Source 
Replaced the fixed 3x interval multiplier with an adaptive threshold
based on the configured interval duration. Health states (`Healthy`,
`Degraded`, `Unhealthy`) were redefined to reflect this change.

- Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower
  intervals (≥10s), ensuring faster problem detection for longer
  intervals and tolerance for jitter in shorter intervals.
- Updated `CheckHealthAsync` to calculate `maxAllowedDelay` using the
  adaptive multiplier.
- Added `Multiplier` and `MaxAllowedDelay` to diagnostic data for
  better monitoring and debugging.
- Improved documentation and comments to explain the new logic.
This commit is contained in:
tekh
2026-07-13 13:54:58 +02:00
parent bc881bf70f
commit 275b06fedb
1 changed files with 17 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs
View File

@@ -18,12 +18,14 @@ namespace ECMJobRunner.WebCron.ProfileWorker;
/// <item><description>Tracks health status based on success/failure patterns</description></item> /// <item><description>Tracks health status based on success/failure patterns</description></item>
/// <item><description>Gracefully handles cancellation during application shutdown</description></item> /// <item><description>Gracefully handles cancellation during application shutdown</description></item>
/// </list> /// </list>
/// Health states: /// Health states (with adaptive thresholds):
/// <list type="bullet"> /// <list type="bullet">
/// <item><description><b>Healthy</b>: Last successful run within 3x interval</description></item> /// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
/// <item><description><b>Degraded</b>: 1-2 consecutive failures but within time limit</description></item> /// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item>
/// <item><description><b>Unhealthy</b>: No success for 3x interval or 3+ consecutive failures</description></item> /// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
/// </list> /// </list>
/// The adaptive multiplier ensures fast problem detection when using longer intervals (e.g., 60s interval = 90s timeout)
/// while maintaining tolerance for network jitter on short intervals (e.g., 1s interval = 3s timeout).
/// </remarks> /// </remarks>
public class ProfileWorker( public class ProfileWorker(
ILogger<ProfileWorker> Logger, ILogger<ProfileWorker> Logger,
@@ -112,20 +114,24 @@ public class ProfileWorker(
/// <remarks> /// <remarks>
/// Health determination logic: /// Health determination logic:
/// <list type="number"> /// <list type="number">
/// <item><description><b>Unhealthy</b>: Time since last success exceeds 3x the configured interval</description></item> /// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
/// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item> /// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item> /// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
/// </list> /// </list>
/// Includes diagnostic data in the result for monitoring and alerting. /// Adaptive multiplier ensures fast problem detection for longer intervals while maintaining tolerance for short intervals.
/// </remarks> /// </remarks>
public Task<HealthCheckResult> CheckHealthAsync( public Task<HealthCheckResult> CheckHealthAsync(
HealthCheckContext context, HealthCheckContext context,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun; var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3);
// Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
// This ensures faster problem detection when using longer intervals (e.g., 60s)
var multiplier = _options.IntervalMS < 10000 ? 3.0 : 1.5;
var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * multiplier);
// Unhealthy: No successful run for 3x interval // Unhealthy: No successful run within adaptive threshold
if (timeSinceLastSuccess > maxAllowedDelay) if (timeSinceLastSuccess > maxAllowedDelay)
{ {
var data = new Dictionary<string, object> var data = new Dictionary<string, object>
@@ -133,7 +139,9 @@ public class ProfileWorker(
["LastSuccessfulRun"] = _lastSuccessfulRun, ["LastSuccessfulRun"] = _lastSuccessfulRun,
["TimeSinceLastSuccess"] = timeSinceLastSuccess, ["TimeSinceLastSuccess"] = timeSinceLastSuccess,
["ConsecutiveFailures"] = _consecutiveFailures, ["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS ["IntervalMS"] = _options.IntervalMS,
["Multiplier"] = multiplier,
["MaxAllowedDelay"] = maxAllowedDelay
}; };
if (_lastException != null) if (_lastException != null)