feat: Add health check monitoring for ProfileWorker
ProfileWorker changes: - Implement IHealthCheck interface - Replace _workCount with DateTime-based tracking - Track _lastSuccessfulRun, _consecutiveFailures, _lastException - Graceful shutdown handling (OperationCanceledException) - Health states: Healthy, Degraded (1-2 failures), Unhealthy (3x interval) DependencyInjection changes: - Register ProfileWorker as singleton (for health check access) - Use factory pattern for IHostedService registration Program.cs changes: - Add health check service with ProfileWorker - Map /health endpoint (full JSON response with all checks) - Map /health/ready endpoint (filtered by 'ready' tag) - Custom JSON response writer with detailed metrics
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace ECMJobRunner.WebCron.ProfileWorker;
|
||||
@@ -5,35 +6,114 @@ namespace ECMJobRunner.WebCron.ProfileWorker;
|
||||
public class ProfileWorker(
|
||||
ILogger<ProfileWorker> Logger,
|
||||
IServiceScopeFactory ScopeFactory,
|
||||
IOptions<ProfileWorkerOptions> Options) : BackgroundService
|
||||
IOptions<ProfileWorkerOptions> Options) : BackgroundService, IHealthCheck
|
||||
{
|
||||
private readonly ProfileWorkerOptions _options = Options.Value;
|
||||
|
||||
private int _workCount = 0;
|
||||
// Health check state
|
||||
private DateTime _lastSuccessfulRun = DateTime.UtcNow;
|
||||
private int _consecutiveFailures = 0;
|
||||
private Exception? _lastException = null;
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
Logger.LogInformation("ProfileWorker started");
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
_workCount++;
|
||||
if (Logger.IsEnabled(LogLevel.Information))
|
||||
{
|
||||
Logger.LogInformation("Worker running {workCount} at: {time}", _workCount, DateTimeOffset.Now);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline)
|
||||
using var scope = ScopeFactory.CreateScope();
|
||||
var work = scope.ServiceProvider.GetRequiredService<ProfileWork>();
|
||||
await work.ExecuteAsync(stoppingToken);
|
||||
|
||||
// Success - update health state
|
||||
_lastSuccessfulRun = DateTime.UtcNow;
|
||||
_consecutiveFailures = 0;
|
||||
_lastException = null;
|
||||
|
||||
if (Logger.IsEnabled(LogLevel.Debug))
|
||||
{
|
||||
Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
// Graceful shutdown - app is stopping
|
||||
Logger.LogInformation("ProfileWorker stopping due to cancellation request");
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex, "An unexpected error occurred in ProfileWorker. Work count: {workCount}", _workCount);
|
||||
// Unexpected error - track for health check
|
||||
_consecutiveFailures++;
|
||||
_lastException = ex;
|
||||
|
||||
Logger.LogError(ex,
|
||||
"An unexpected error occurred in ProfileWorker (consecutive failures: {failures})",
|
||||
_consecutiveFailures);
|
||||
}
|
||||
|
||||
await Task.Delay(_options.IntervalMS, stoppingToken);
|
||||
}
|
||||
|
||||
Logger.LogInformation("ProfileWorker stopped");
|
||||
}
|
||||
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
|
||||
var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3);
|
||||
|
||||
// Unhealthy: No successful run for 3x interval
|
||||
if (timeSinceLastSuccess > maxAllowedDelay)
|
||||
{
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
["LastSuccessfulRun"] = _lastSuccessfulRun,
|
||||
["TimeSinceLastSuccess"] = timeSinceLastSuccess,
|
||||
["ConsecutiveFailures"] = _consecutiveFailures,
|
||||
["IntervalMS"] = _options.IntervalMS
|
||||
};
|
||||
|
||||
if (_lastException != null)
|
||||
{
|
||||
data["LastException"] = _lastException.Message;
|
||||
}
|
||||
|
||||
return Task.FromResult(HealthCheckResult.Unhealthy(
|
||||
$"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)",
|
||||
_lastException,
|
||||
data
|
||||
));
|
||||
}
|
||||
|
||||
// Degraded: 1-2 consecutive failures but within time limit
|
||||
if (_consecutiveFailures > 0)
|
||||
{
|
||||
var data = new Dictionary<string, object>
|
||||
{
|
||||
["LastSuccessfulRun"] = _lastSuccessfulRun,
|
||||
["ConsecutiveFailures"] = _consecutiveFailures
|
||||
};
|
||||
|
||||
return Task.FromResult(HealthCheckResult.Degraded(
|
||||
$"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational",
|
||||
null,
|
||||
data
|
||||
));
|
||||
}
|
||||
|
||||
// Healthy
|
||||
return Task.FromResult(HealthCheckResult.Healthy(
|
||||
"ProfileWorker is running normally",
|
||||
new Dictionary<string, object>
|
||||
{
|
||||
["LastSuccessfulRun"] = _lastSuccessfulRun
|
||||
}
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user