AppStd/ECMJobRunner
8
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Improve ProfileWorker health-check logic and initialization

Browse Source 
Refactor the `ProfileWorker` class to enhance health-check handling, including support for an uninitialized state. The `_lastSuccessfulRun` field was changed to a nullable `DateTime?` to represent when the service has not yet completed its first successful run.

Introduce a new health state, **Degraded (Initializing)**, to indicate the service is starting up. Update the `CheckHealthAsync` method to handle this state and return a `HealthCheckResult.Degraded` with relevant metadata.

Refine health-check logic to safely access `_lastSuccessfulRun.Value` only when initialized. Update documentation to clarify health status conditions and retain adaptive multiplier logic for detecting issues based on interval length.

Enhance `HealthCheckResult` metadata with additional details, including `LastSuccessfulRun`, time since last success, and consecutive failures. Improve comments and descriptions for better clarity.
This commit is contained in:
tekh
2026-07-13 14:03:49 +02:00
parent 275b06fedb
commit 9537954626
1 changed files with 34 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs
View File

@@ -20,6 +20,7 @@ namespace ECMJobRunner.WebCron.ProfileWorker;
/// </list>
/// Health states (with adaptive thresholds):
/// <list type="bullet">
/// <item><description><b>Degraded (Initializing)</b>: Service starting up, waiting for first successful run</description></item>
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
/// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item>
/// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
@@ -35,7 +36,7 @@ public class ProfileWorker(
private readonly ProfileWorkerOptions _options = Options.Value;
// Health check state
private DateTime _lastSuccessfulRun = DateTime.UtcNow;
private DateTime? _lastSuccessfulRun = null; // null = not run yet
private int _consecutiveFailures = 0;
private Exception? _lastException = null;
@@ -107,13 +108,14 @@ public class ProfileWorker(
/// A <see cref="HealthCheckResult"/> indicating the current health status:
/// <list type="bullet">
/// <item><description><see cref="HealthStatus.Healthy"/> - Service running normally</description></item>
/// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational</description></item>
/// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational, or service initializing</description></item>
/// <item><description><see cref="HealthStatus.Unhealthy"/> - No successful run for extended period</description></item>
/// </list>
/// </returns>
/// <remarks>
/// Health determination logic:
/// <list type="number">
/// <item><description><b>Degraded (Initializing)</b>: Service has not completed its first successful run yet</description></item>
/// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
/// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
@@ -124,7 +126,32 @@ public class ProfileWorker(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
// Service hasn't completed its first successful run yet
if (_lastSuccessfulRun == null)
{
var data = new Dictionary<string, object>
{
["Status"] = "Initializing",
["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS
};
if (_lastException != null)
{
data["LastException"] = _lastException.Message;
}
// Degraded during initialization (not unhealthy, service is starting up)
return Task.FromResult(HealthCheckResult.Degraded(
_consecutiveFailures > 0
? $"ProfileWorker initializing with {_consecutiveFailures} failure(s)"
: "ProfileWorker is initializing, waiting for first successful run",
_lastException,
data
));
}
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun.Value;
// Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
// This ensures faster problem detection when using longer intervals (e.g., 60s)
@@ -136,7 +163,7 @@ public class ProfileWorker(
{
var data = new Dictionary<string, object>
{
["LastSuccessfulRun"] = _lastSuccessfulRun,
["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
["TimeSinceLastSuccess"] = timeSinceLastSuccess,
["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS,
@@ -156,12 +183,12 @@ public class ProfileWorker(
));
}
// Degraded: 1-2 consecutive failures but within time limit
// Degraded: 1+ consecutive failures but within time limit
if (_consecutiveFailures > 0)
{
var data = new Dictionary<string, object>
{
["LastSuccessfulRun"] = _lastSuccessfulRun,
["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
["ConsecutiveFailures"] = _consecutiveFailures
};
@@ -177,7 +204,7 @@ public class ProfileWorker(
"ProfileWorker is running normally",
new Dictionary<string, object>
{
["LastSuccessfulRun"] = _lastSuccessfulRun
["LastSuccessfulRun"] = _lastSuccessfulRun.Value
}
));
}