Improve ProfileWorker health-check logic and initialization
Refactor the `ProfileWorker` class to enhance health-check handling, including support for an uninitialized state. The `_lastSuccessfulRun` field was changed to a nullable `DateTime?` to represent when the service has not yet completed its first successful run. Introduce a new health state, **Degraded (Initializing)**, to indicate the service is starting up. Update the `CheckHealthAsync` method to handle this state and return a `HealthCheckResult.Degraded` with relevant metadata. Refine health-check logic to safely access `_lastSuccessfulRun.Value` only when initialized. Update documentation to clarify health status conditions and retain adaptive multiplier logic for detecting issues based on interval length. Enhance `HealthCheckResult` metadata with additional details, including `LastSuccessfulRun`, time since last success, and consecutive failures. Improve comments and descriptions for better clarity.
This commit is contained in:
@@ -20,6 +20,7 @@ namespace ECMJobRunner.WebCron.ProfileWorker;
|
|||||||
/// </list>
|
/// </list>
|
||||||
/// Health states (with adaptive thresholds):
|
/// Health states (with adaptive thresholds):
|
||||||
/// <list type="bullet">
|
/// <list type="bullet">
|
||||||
|
/// <item><description><b>Degraded (Initializing)</b>: Service starting up, waiting for first successful run</description></item>
|
||||||
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
|
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
|
||||||
/// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item>
|
/// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item>
|
||||||
/// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)</description></item>
|
/// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)</description></item>
|
||||||
@@ -35,7 +36,7 @@ public class ProfileWorker(
|
|||||||
private readonly ProfileWorkerOptions _options = Options.Value;
|
private readonly ProfileWorkerOptions _options = Options.Value;
|
||||||
|
|
||||||
// Health check state
|
// Health check state
|
||||||
private DateTime _lastSuccessfulRun = DateTime.UtcNow;
|
private DateTime? _lastSuccessfulRun = null; // null = not run yet
|
||||||
private int _consecutiveFailures = 0;
|
private int _consecutiveFailures = 0;
|
||||||
private Exception? _lastException = null;
|
private Exception? _lastException = null;
|
||||||
|
|
||||||
@@ -107,13 +108,14 @@ public class ProfileWorker(
|
|||||||
/// A <see cref="HealthCheckResult"/> indicating the current health status:
|
/// A <see cref="HealthCheckResult"/> indicating the current health status:
|
||||||
/// <list type="bullet">
|
/// <list type="bullet">
|
||||||
/// <item><description><see cref="HealthStatus.Healthy"/> - Service running normally</description></item>
|
/// <item><description><see cref="HealthStatus.Healthy"/> - Service running normally</description></item>
|
||||||
/// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational</description></item>
|
/// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational, or service initializing</description></item>
|
||||||
/// <item><description><see cref="HealthStatus.Unhealthy"/> - No successful run for extended period</description></item>
|
/// <item><description><see cref="HealthStatus.Unhealthy"/> - No successful run for extended period</description></item>
|
||||||
/// </list>
|
/// </list>
|
||||||
/// </returns>
|
/// </returns>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// Health determination logic:
|
/// Health determination logic:
|
||||||
/// <list type="number">
|
/// <list type="number">
|
||||||
|
/// <item><description><b>Degraded (Initializing)</b>: Service has not completed its first successful run yet</description></item>
|
||||||
/// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)</description></item>
|
/// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)</description></item>
|
||||||
/// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
|
/// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
|
||||||
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
|
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
|
||||||
@@ -124,7 +126,32 @@ public class ProfileWorker(
|
|||||||
HealthCheckContext context,
|
HealthCheckContext context,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
|
// Service hasn't completed its first successful run yet
|
||||||
|
if (_lastSuccessfulRun == null)
|
||||||
|
{
|
||||||
|
var data = new Dictionary<string, object>
|
||||||
|
{
|
||||||
|
["Status"] = "Initializing",
|
||||||
|
["ConsecutiveFailures"] = _consecutiveFailures,
|
||||||
|
["IntervalMS"] = _options.IntervalMS
|
||||||
|
};
|
||||||
|
|
||||||
|
if (_lastException != null)
|
||||||
|
{
|
||||||
|
data["LastException"] = _lastException.Message;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Degraded during initialization (not unhealthy, service is starting up)
|
||||||
|
return Task.FromResult(HealthCheckResult.Degraded(
|
||||||
|
_consecutiveFailures > 0
|
||||||
|
? $"ProfileWorker initializing with {_consecutiveFailures} failure(s)"
|
||||||
|
: "ProfileWorker is initializing, waiting for first successful run",
|
||||||
|
_lastException,
|
||||||
|
data
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun.Value;
|
||||||
|
|
||||||
// Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
|
// Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
|
||||||
// This ensures faster problem detection when using longer intervals (e.g., 60s)
|
// This ensures faster problem detection when using longer intervals (e.g., 60s)
|
||||||
@@ -136,7 +163,7 @@ public class ProfileWorker(
|
|||||||
{
|
{
|
||||||
var data = new Dictionary<string, object>
|
var data = new Dictionary<string, object>
|
||||||
{
|
{
|
||||||
["LastSuccessfulRun"] = _lastSuccessfulRun,
|
["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
|
||||||
["TimeSinceLastSuccess"] = timeSinceLastSuccess,
|
["TimeSinceLastSuccess"] = timeSinceLastSuccess,
|
||||||
["ConsecutiveFailures"] = _consecutiveFailures,
|
["ConsecutiveFailures"] = _consecutiveFailures,
|
||||||
["IntervalMS"] = _options.IntervalMS,
|
["IntervalMS"] = _options.IntervalMS,
|
||||||
@@ -156,12 +183,12 @@ public class ProfileWorker(
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Degraded: 1-2 consecutive failures but within time limit
|
// Degraded: 1+ consecutive failures but within time limit
|
||||||
if (_consecutiveFailures > 0)
|
if (_consecutiveFailures > 0)
|
||||||
{
|
{
|
||||||
var data = new Dictionary<string, object>
|
var data = new Dictionary<string, object>
|
||||||
{
|
{
|
||||||
["LastSuccessfulRun"] = _lastSuccessfulRun,
|
["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
|
||||||
["ConsecutiveFailures"] = _consecutiveFailures
|
["ConsecutiveFailures"] = _consecutiveFailures
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -177,7 +204,7 @@ public class ProfileWorker(
|
|||||||
"ProfileWorker is running normally",
|
"ProfileWorker is running normally",
|
||||||
new Dictionary<string, object>
|
new Dictionary<string, object>
|
||||||
{
|
{
|
||||||
["LastSuccessfulRun"] = _lastSuccessfulRun
|
["LastSuccessfulRun"] = _lastSuccessfulRun.Value
|
||||||
}
|
}
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user