using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
namespace ECMJobRunner.WebCron.ProfileWorker;
///
/// Background service that continuously synchronizes active profiles with Hangfire recurring jobs.
/// Implements health check monitoring to track service status and failure conditions.
///
/// Logger for diagnostic output.
/// Factory for creating service scopes (required for scoped service resolution).
/// Configuration options for interval timing.
///
/// This service runs on a configurable interval (default 1 second) and performs the following:
///
/// - Fetches active profiles from the database
/// - Synchronizes profiles with Hangfire recurring jobs
/// - Tracks health status based on success/failure patterns
/// - Gracefully handles cancellation during application shutdown
///
/// Health states:
///
/// - Healthy: Last successful run within 3x interval
/// - Degraded: 1-2 consecutive failures but within time limit
/// - Unhealthy: No success for 3x interval or 3+ consecutive failures
///
///
public class ProfileWorker(
ILogger Logger,
IServiceScopeFactory ScopeFactory,
IOptions Options) : BackgroundService, IHealthCheck
{
private readonly ProfileWorkerOptions _options = Options.Value;
// Health check state
private DateTime _lastSuccessfulRun = DateTime.UtcNow;
private int _consecutiveFailures = 0;
private Exception? _lastException = null;
///
/// Background service execution loop that synchronizes profiles on a configurable interval.
///
/// Cancellation token for graceful shutdown.
/// A task representing the background execution.
///
/// The loop continues until:
///
/// - Application shutdown is requested (via stoppingToken)
/// - An unhandled exception causes service failure
///
/// Uses scoped services for each iteration to ensure proper lifetime management.
///
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
Logger.LogInformation("ProfileWorker started");
while (!stoppingToken.IsCancellationRequested)
{
try
{
// Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline)
using var scope = ScopeFactory.CreateScope();
var work = scope.ServiceProvider.GetRequiredService();
await work.ExecuteAsync(stoppingToken);
// Success - update health state
_lastSuccessfulRun = DateTime.UtcNow;
_consecutiveFailures = 0;
_lastException = null;
if (Logger.IsEnabled(LogLevel.Debug))
{
Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// Graceful shutdown - app is stopping
Logger.LogInformation("ProfileWorker stopping due to cancellation request");
break;
}
catch (Exception ex)
{
// Unexpected error - track for health check
_consecutiveFailures++;
_lastException = ex;
Logger.LogError(ex,
"An unexpected error occurred in ProfileWorker (consecutive failures: {failures})",
_consecutiveFailures);
}
await Task.Delay(_options.IntervalMS, stoppingToken);
}
Logger.LogInformation("ProfileWorker stopped");
}
///
/// Performs a health check by evaluating recent success/failure patterns.
///
/// Health check context (unused).
/// Cancellation token.
///
/// A indicating the current health status:
///
/// - - Service running normally
/// - - Recent failures but still operational
/// - - No successful run for extended period
///
///
///
/// Health determination logic:
///
/// - Unhealthy: Time since last success exceeds 3x the configured interval
/// - Degraded: 1+ consecutive failures within time threshold
/// - Healthy: Recent successful run with no failures
///
/// Includes diagnostic data in the result for monitoring and alerting.
///
public Task CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3);
// Unhealthy: No successful run for 3x interval
if (timeSinceLastSuccess > maxAllowedDelay)
{
var data = new Dictionary
{
["LastSuccessfulRun"] = _lastSuccessfulRun,
["TimeSinceLastSuccess"] = timeSinceLastSuccess,
["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS
};
if (_lastException != null)
{
data["LastException"] = _lastException.Message;
}
return Task.FromResult(HealthCheckResult.Unhealthy(
$"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)",
_lastException,
data
));
}
// Degraded: 1-2 consecutive failures but within time limit
if (_consecutiveFailures > 0)
{
var data = new Dictionary
{
["LastSuccessfulRun"] = _lastSuccessfulRun,
["ConsecutiveFailures"] = _consecutiveFailures
};
return Task.FromResult(HealthCheckResult.Degraded(
$"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational",
null,
data
));
}
// Healthy
return Task.FromResult(HealthCheckResult.Healthy(
"ProfileWorker is running normally",
new Dictionary
{
["LastSuccessfulRun"] = _lastSuccessfulRun
}
));
}
}