using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;
namespace ECMJobRunner.WebCron.ProfileWorker;
///
/// Background service that continuously synchronizes active profiles with Hangfire recurring jobs.
/// Implements health check monitoring to track service status and failure conditions.
///
/// Logger for diagnostic output.
/// Factory for creating service scopes (required for scoped service resolution).
/// Configuration options for interval timing.
///
/// This service runs on a configurable interval (default 1 second) and performs the following:
///
/// - Fetches active profiles from the database
/// - Synchronizes profiles with Hangfire recurring jobs
/// - Tracks health status based on success/failure patterns
/// - Gracefully handles cancellation during application shutdown
///
/// Health states (with adaptive thresholds):
///
/// - Degraded (Initializing): Service starting up, waiting for first successful run
/// - Healthy: Recent successful run with no failures
/// - Degraded: 1+ consecutive failures but within time threshold
/// - Unhealthy: No success within adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)
///
/// The adaptive multiplier ensures fast problem detection when using longer intervals (e.g., 60s interval = 90s timeout)
/// while maintaining tolerance for network jitter on short intervals (e.g., 1s interval = 3s timeout).
///
public class ProfileWorker(
ILogger Logger,
IServiceScopeFactory ScopeFactory,
IOptions Options) : BackgroundService, IHealthCheck
{
private readonly ProfileWorkerOptions _options = Options.Value;
// Health check state
private DateTime? _lastSuccessfulRun = null; // null = not run yet
private int _consecutiveFailures = 0;
private Exception? _lastException = null;
///
/// Background service execution loop that synchronizes profiles on a configurable interval.
///
/// Cancellation token for graceful shutdown.
/// A task representing the background execution.
///
/// The loop continues until:
///
/// - Application shutdown is requested (via stoppingToken)
/// - An unhandled exception causes service failure
///
/// Uses scoped services for each iteration to ensure proper lifetime management.
///
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
Logger.LogInformation("ProfileWorker started");
while (!stoppingToken.IsCancellationRequested)
{
try
{
// Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline)
using var scope = ScopeFactory.CreateScope();
var work = scope.ServiceProvider.GetRequiredService();
await work.ExecuteAsync(stoppingToken);
// Success - update health state
_lastSuccessfulRun = DateTime.Now;
_consecutiveFailures = 0;
_lastException = null;
if (Logger.IsEnabled(LogLevel.Debug))
{
Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun);
}
}
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
{
// Graceful shutdown - app is stopping
Logger.LogInformation("ProfileWorker stopping due to cancellation request");
break;
}
catch (Exception ex)
{
// Unexpected error - track for health check
_consecutiveFailures++;
_lastException = ex;
Logger.LogError(ex,
"An unexpected error occurred in ProfileWorker (consecutive failures: {failures})",
_consecutiveFailures);
}
await Task.Delay(_options.IntervalMS, stoppingToken);
}
Logger.LogInformation("ProfileWorker stopped");
}
///
/// Performs a health check by evaluating recent success/failure patterns.
///
/// Health check context (unused).
/// Cancellation token.
///
/// A indicating the current health status:
///
/// - - Service running normally
/// - - Recent failures but still operational, or service initializing
/// - - No successful run for extended period
///
///
///
/// Health determination logic:
///
/// - Degraded (Initializing): Service has not completed its first successful run yet
/// - Unhealthy: Time since last success exceeds adaptive threshold (3x interval for fast intervals <10s, 1.5x for slower intervals)
/// - Degraded: 1+ consecutive failures within time threshold
/// - Healthy: Recent successful run with no failures
///
/// Adaptive multiplier ensures fast problem detection for longer intervals while maintaining tolerance for short intervals.
///
public Task CheckHealthAsync(
HealthCheckContext context,
CancellationToken cancellationToken = default)
{
// Service hasn't completed its first successful run yet
if (_lastSuccessfulRun == null)
{
var data = new Dictionary
{
["Status"] = "Initializing",
["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS
};
if (_lastException != null)
{
data["LastException"] = _lastException.Message;
}
// Degraded during initialization (not unhealthy, service is starting up)
return Task.FromResult(HealthCheckResult.Degraded(
_consecutiveFailures > 0
? $"ProfileWorker initializing with {_consecutiveFailures} failure(s)"
: "ProfileWorker is initializing, waiting for first successful run",
_lastException,
data
));
}
var timeSinceLastSuccess = DateTime.Now - _lastSuccessfulRun.Value;
// Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
// This ensures faster problem detection when using longer intervals (e.g., 60s)
var multiplier = _options.IntervalMS < 10000 ? 3.0 : 1.5;
var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * multiplier);
// Unhealthy: No successful run within adaptive threshold
if (timeSinceLastSuccess > maxAllowedDelay)
{
var data = new Dictionary
{
["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
["TimeSinceLastSuccess"] = timeSinceLastSuccess,
["ConsecutiveFailures"] = _consecutiveFailures,
["IntervalMS"] = _options.IntervalMS,
["Multiplier"] = multiplier,
["MaxAllowedDelay"] = maxAllowedDelay
};
if (_lastException != null)
{
data["LastException"] = _lastException.Message;
}
return Task.FromResult(HealthCheckResult.Unhealthy(
$"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)",
_lastException,
data
));
}
// Degraded: 1+ consecutive failures but within time limit
if (_consecutiveFailures > 0)
{
var data = new Dictionary
{
["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
["ConsecutiveFailures"] = _consecutiveFailures
};
return Task.FromResult(HealthCheckResult.Degraded(
$"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational",
null,
data
));
}
// Healthy
return Task.FromResult(HealthCheckResult.Healthy(
"ProfileWorker is running normally",
new Dictionary
{
["LastSuccessfulRun"] = _lastSuccessfulRun.Value
}
));
}
}