Improve ProfileWorker health-check logic and initialization

Refactor the `ProfileWorker` class to enhance health-check handling, including support for an uninitialized state. The `_lastSuccessfulRun` field was changed to a nullable `DateTime?` to represent when the service has not yet completed its first successful run.

Introduce a new health state, **Degraded (Initializing)**, to indicate the service is starting up. Update the `CheckHealthAsync` method to handle this state and return a `HealthCheckResult.Degraded` with relevant metadata.

Refine health-check logic to safely access `_lastSuccessfulRun.Value` only when initialized. Update documentation to clarify health status conditions and retain adaptive multiplier logic for detecting issues based on interval length.

Enhance `HealthCheckResult` metadata with additional details, including `LastSuccessfulRun`, time since last success, and consecutive failures. Improve comments and descriptions for better clarity.
This commit is contained in:
2026-07-13 14:03:49 +02:00
parent 275b06fedb
commit 9537954626

View File

@@ -20,6 +20,7 @@ namespace ECMJobRunner.WebCron.ProfileWorker;
/// </list> /// </list>
/// Health states (with adaptive thresholds): /// Health states (with adaptive thresholds):
/// <list type="bullet"> /// <list type="bullet">
/// <item><description><b>Degraded (Initializing)</b>: Service starting up, waiting for first successful run</description></item>
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item> /// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
/// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item> /// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item>
/// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item> /// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
@@ -35,7 +36,7 @@ public class ProfileWorker(
private readonly ProfileWorkerOptions _options = Options.Value; private readonly ProfileWorkerOptions _options = Options.Value;
// Health check state // Health check state
private DateTime _lastSuccessfulRun = DateTime.UtcNow; private DateTime? _lastSuccessfulRun = null; // null = not run yet
private int _consecutiveFailures = 0; private int _consecutiveFailures = 0;
private Exception? _lastException = null; private Exception? _lastException = null;
@@ -107,13 +108,14 @@ public class ProfileWorker(
/// A <see cref="HealthCheckResult"/> indicating the current health status: /// A <see cref="HealthCheckResult"/> indicating the current health status:
/// <list type="bullet"> /// <list type="bullet">
/// <item><description><see cref="HealthStatus.Healthy"/> - Service running normally</description></item> /// <item><description><see cref="HealthStatus.Healthy"/> - Service running normally</description></item>
/// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational</description></item> /// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational, or service initializing</description></item>
/// <item><description><see cref="HealthStatus.Unhealthy"/> - No successful run for extended period</description></item> /// <item><description><see cref="HealthStatus.Unhealthy"/> - No successful run for extended period</description></item>
/// </list> /// </list>
/// </returns> /// </returns>
/// <remarks> /// <remarks>
/// Health determination logic: /// Health determination logic:
/// <list type="number"> /// <list type="number">
/// <item><description><b>Degraded (Initializing)</b>: Service has not completed its first successful run yet</description></item>
/// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item> /// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
/// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item> /// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item> /// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
@@ -124,7 +126,32 @@ public class ProfileWorker(
HealthCheckContext context, HealthCheckContext context,
CancellationToken cancellationToken = default) CancellationToken cancellationToken = default)
{ {
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun; // Service hasn't completed its first successful run yet
if (_lastSuccessfulRun == null)
{
var data = new Dictionary<string, object>
{
["Status"] = "Initializing",
["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS
};
if (_lastException != null)
{
data["LastException"] = _lastException.Message;
}
// Degraded during initialization (not unhealthy, service is starting up)
return Task.FromResult(HealthCheckResult.Degraded(
_consecutiveFailures > 0
? $"ProfileWorker initializing with {_consecutiveFailures} failure(s)"
: "ProfileWorker is initializing, waiting for first successful run",
_lastException,
data
));
}
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun.Value;
// Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals // Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
// This ensures faster problem detection when using longer intervals (e.g., 60s) // This ensures faster problem detection when using longer intervals (e.g., 60s)
@@ -136,7 +163,7 @@ public class ProfileWorker(
{ {
var data = new Dictionary<string, object> var data = new Dictionary<string, object>
{ {
["LastSuccessfulRun"] = _lastSuccessfulRun, ["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
["TimeSinceLastSuccess"] = timeSinceLastSuccess, ["TimeSinceLastSuccess"] = timeSinceLastSuccess,
["ConsecutiveFailures"] = _consecutiveFailures, ["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS, ["IntervalMS"] = _options.IntervalMS,
@@ -156,12 +183,12 @@ public class ProfileWorker(
)); ));
} }
// Degraded: 1-2 consecutive failures but within time limit // Degraded: 1+ consecutive failures but within time limit
if (_consecutiveFailures > 0) if (_consecutiveFailures > 0)
{ {
var data = new Dictionary<string, object> var data = new Dictionary<string, object>
{ {
["LastSuccessfulRun"] = _lastSuccessfulRun, ["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
["ConsecutiveFailures"] = _consecutiveFailures ["ConsecutiveFailures"] = _consecutiveFailures
}; };
@@ -177,7 +204,7 @@ public class ProfileWorker(
"ProfileWorker is running normally", "ProfileWorker is running normally",
new Dictionary<string, object> new Dictionary<string, object>
{ {
["LastSuccessfulRun"] = _lastSuccessfulRun ["LastSuccessfulRun"] = _lastSuccessfulRun.Value
} }
)); ));
} }