using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; namespace ECMJobRunner.WebCron.ProfileWorker; /// /// Background service that continuously synchronizes active profiles with Hangfire recurring jobs. /// Implements health check monitoring to track service status and failure conditions. /// /// Logger for diagnostic output. /// Factory for creating service scopes (required for scoped service resolution). /// Configuration options for interval timing. /// /// This service runs on a configurable interval (default 1 second) and performs the following: /// /// Fetches active profiles from the database /// Synchronizes profiles with Hangfire recurring jobs /// Tracks health status based on success/failure patterns /// Gracefully handles cancellation during application shutdown /// /// Health states: /// /// Healthy: Last successful run within 3x interval /// Degraded: 1-2 consecutive failures but within time limit /// Unhealthy: No success for 3x interval or 3+ consecutive failures /// /// public class ProfileWorker( ILogger Logger, IServiceScopeFactory ScopeFactory, IOptions Options) : BackgroundService, IHealthCheck { private readonly ProfileWorkerOptions _options = Options.Value; // Health check state private DateTime _lastSuccessfulRun = DateTime.UtcNow; private int _consecutiveFailures = 0; private Exception? _lastException = null; /// /// Background service execution loop that synchronizes profiles on a configurable interval. /// /// Cancellation token for graceful shutdown. /// A task representing the background execution. /// /// The loop continues until: /// /// Application shutdown is requested (via stoppingToken) /// An unhandled exception causes service failure /// /// Uses scoped services for each iteration to ensure proper lifetime management. /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { Logger.LogInformation("ProfileWorker started"); while (!stoppingToken.IsCancellationRequested) { try { // Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline) using var scope = ScopeFactory.CreateScope(); var work = scope.ServiceProvider.GetRequiredService(); await work.ExecuteAsync(stoppingToken); // Success - update health state _lastSuccessfulRun = DateTime.UtcNow; _consecutiveFailures = 0; _lastException = null; if (Logger.IsEnabled(LogLevel.Debug)) { Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun); } } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { // Graceful shutdown - app is stopping Logger.LogInformation("ProfileWorker stopping due to cancellation request"); break; } catch (Exception ex) { // Unexpected error - track for health check _consecutiveFailures++; _lastException = ex; Logger.LogError(ex, "An unexpected error occurred in ProfileWorker (consecutive failures: {failures})", _consecutiveFailures); } await Task.Delay(_options.IntervalMS, stoppingToken); } Logger.LogInformation("ProfileWorker stopped"); } /// /// Performs a health check by evaluating recent success/failure patterns. /// /// Health check context (unused). /// Cancellation token. /// /// A indicating the current health status: /// /// - Service running normally /// - Recent failures but still operational /// - No successful run for extended period /// /// /// /// Health determination logic: /// /// Unhealthy: Time since last success exceeds 3x the configured interval /// Degraded: 1+ consecutive failures within time threshold /// Healthy: Recent successful run with no failures /// /// Includes diagnostic data in the result for monitoring and alerting. /// public Task CheckHealthAsync( HealthCheckContext context, CancellationToken cancellationToken = default) { var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun; var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3); // Unhealthy: No successful run for 3x interval if (timeSinceLastSuccess > maxAllowedDelay) { var data = new Dictionary { ["LastSuccessfulRun"] = _lastSuccessfulRun, ["TimeSinceLastSuccess"] = timeSinceLastSuccess, ["ConsecutiveFailures"] = _consecutiveFailures, ["IntervalMS"] = _options.IntervalMS }; if (_lastException != null) { data["LastException"] = _lastException.Message; } return Task.FromResult(HealthCheckResult.Unhealthy( $"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)", _lastException, data )); } // Degraded: 1-2 consecutive failures but within time limit if (_consecutiveFailures > 0) { var data = new Dictionary { ["LastSuccessfulRun"] = _lastSuccessfulRun, ["ConsecutiveFailures"] = _consecutiveFailures }; return Task.FromResult(HealthCheckResult.Degraded( $"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational", null, data )); } // Healthy return Task.FromResult(HealthCheckResult.Healthy( "ProfileWorker is running normally", new Dictionary { ["LastSuccessfulRun"] = _lastSuccessfulRun } )); } }