ECMJobRunner/ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs

using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;

namespace ECMJobRunner.WebCron.ProfileWorker;

public class ProfileWorker(
    ILogger<ProfileWorker> Logger, 
    IServiceScopeFactory ScopeFactory,
    IOptions<ProfileWorkerOptions> Options) : BackgroundService, IHealthCheck
{
    private readonly ProfileWorkerOptions _options = Options.Value;

    // Health check state
    private DateTime _lastSuccessfulRun = DateTime.UtcNow;
    private int _consecutiveFailures = 0;
    private Exception? _lastException = null;

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        Logger.LogInformation("ProfileWorker started");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                // Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline)
                using var scope = ScopeFactory.CreateScope();
                var work = scope.ServiceProvider.GetRequiredService<ProfileWork>();
                await work.ExecuteAsync(stoppingToken);

                // Success - update health state
                _lastSuccessfulRun = DateTime.UtcNow;
                _consecutiveFailures = 0;
                _lastException = null;

                if (Logger.IsEnabled(LogLevel.Debug))
                {
                    Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun);
                }
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                // Graceful shutdown - app is stopping
                Logger.LogInformation("ProfileWorker stopping due to cancellation request");
                break;
            }
            catch (Exception ex)
            {
                // Unexpected error - track for health check
                _consecutiveFailures++;
                _lastException = ex;

                Logger.LogError(ex, 
                    "An unexpected error occurred in ProfileWorker (consecutive failures: {failures})", 
                    _consecutiveFailures);
            }

            await Task.Delay(_options.IntervalMS, stoppingToken);
        }

        Logger.LogInformation("ProfileWorker stopped");
    }

    public Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context, 
        CancellationToken cancellationToken = default)
    {
        var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
        var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3);

        // Unhealthy: No successful run for 3x interval
        if (timeSinceLastSuccess > maxAllowedDelay)
        {
            var data = new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun,
                ["TimeSinceLastSuccess"] = timeSinceLastSuccess,
                ["ConsecutiveFailures"] = _consecutiveFailures,
                ["IntervalMS"] = _options.IntervalMS
            };

            if (_lastException != null)
            {
                data["LastException"] = _lastException.Message;
            }

            return Task.FromResult(HealthCheckResult.Unhealthy(
                $"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)",
                _lastException,
                data
            ));
        }

        // Degraded: 1-2 consecutive failures but within time limit
        if (_consecutiveFailures > 0)
        {
            var data = new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun,
                ["ConsecutiveFailures"] = _consecutiveFailures
            };

            return Task.FromResult(HealthCheckResult.Degraded(
                $"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational",
                null,
                data
            ));
        }

        // Healthy
        return Task.FromResult(HealthCheckResult.Healthy(
            "ProfileWorker is running normally",
            new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun
            }
        ));
    }
}