ECMJobRunner/ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs

using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;

namespace ECMJobRunner.WebCron.ProfileWorker;

/// <summary>
/// Background service that continuously synchronizes active profiles with Hangfire recurring jobs.
/// Implements health check monitoring to track service status and failure conditions.
/// </summary>
/// <param name="Logger">Logger for diagnostic output.</param>
/// <param name="ScopeFactory">Factory for creating service scopes (required for scoped service resolution).</param>
/// <param name="Options">Configuration options for interval timing.</param>
/// <remarks>
/// This service runs on a configurable interval (default 1 second) and performs the following:
/// <list type="bullet">
/// <item><description>Fetches active profiles from the database</description></item>
/// <item><description>Synchronizes profiles with Hangfire recurring jobs</description></item>
/// <item><description>Tracks health status based on success/failure patterns</description></item>
/// <item><description>Gracefully handles cancellation during application shutdown</description></item>
/// </list>
/// Health states:
/// <list type="bullet">
/// <item><description><b>Healthy</b>: Last successful run within 3x interval</description></item>
/// <item><description><b>Degraded</b>: 1-2 consecutive failures but within time limit</description></item>
/// <item><description><b>Unhealthy</b>: No success for 3x interval or 3+ consecutive failures</description></item>
/// </list>
/// </remarks>
public class ProfileWorker(
    ILogger<ProfileWorker> Logger, 
    IServiceScopeFactory ScopeFactory,
    IOptions<ProfileWorkerOptions> Options) : BackgroundService, IHealthCheck
{
    private readonly ProfileWorkerOptions _options = Options.Value;

    // Health check state
    private DateTime _lastSuccessfulRun = DateTime.UtcNow;
    private int _consecutiveFailures = 0;
    private Exception? _lastException = null;

    /// <summary>
    /// Background service execution loop that synchronizes profiles on a configurable interval.
    /// </summary>
    /// <param name="stoppingToken">Cancellation token for graceful shutdown.</param>
    /// <returns>A task representing the background execution.</returns>
    /// <remarks>
    /// The loop continues until:
    /// <list type="bullet">
    /// <item><description>Application shutdown is requested (via stoppingToken)</description></item>
    /// <item><description>An unhandled exception causes service failure</description></item>
    /// </list>
    /// Uses scoped services for each iteration to ensure proper lifetime management.
    /// </remarks>
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        Logger.LogInformation("ProfileWorker started");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                // Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline)
                using var scope = ScopeFactory.CreateScope();
                var work = scope.ServiceProvider.GetRequiredService<ProfileWork>();
                await work.ExecuteAsync(stoppingToken);

                // Success - update health state
                _lastSuccessfulRun = DateTime.UtcNow;
                _consecutiveFailures = 0;
                _lastException = null;

                if (Logger.IsEnabled(LogLevel.Debug))
                {
                    Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun);
                }
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                // Graceful shutdown - app is stopping
                Logger.LogInformation("ProfileWorker stopping due to cancellation request");
                break;
            }
            catch (Exception ex)
            {
                // Unexpected error - track for health check
                _consecutiveFailures++;
                _lastException = ex;

                Logger.LogError(ex, 
                    "An unexpected error occurred in ProfileWorker (consecutive failures: {failures})", 
                    _consecutiveFailures);
            }

            await Task.Delay(_options.IntervalMS, stoppingToken);
        }

        Logger.LogInformation("ProfileWorker stopped");
    }

    /// <summary>
    /// Performs a health check by evaluating recent success/failure patterns.
    /// </summary>
    /// <param name="context">Health check context (unused).</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>
    /// A <see cref="HealthCheckResult"/> indicating the current health status:
    /// <list type="bullet">
    /// <item><description><see cref="HealthStatus.Healthy"/> - Service running normally</description></item>
    /// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational</description></item>
    /// <item><description><see cref="HealthStatus.Unhealthy"/> - No successful run for extended period</description></item>
    /// </list>
    /// </returns>
    /// <remarks>
    /// Health determination logic:
    /// <list type="number">
    /// <item><description><b>Unhealthy</b>: Time since last success exceeds 3x the configured interval</description></item>
    /// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
    /// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
    /// </list>
    /// Includes diagnostic data in the result for monitoring and alerting.
    /// </remarks>
    public Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context, 
        CancellationToken cancellationToken = default)
    {
        var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun;
        var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3);

        // Unhealthy: No successful run for 3x interval
        if (timeSinceLastSuccess > maxAllowedDelay)
        {
            var data = new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun,
                ["TimeSinceLastSuccess"] = timeSinceLastSuccess,
                ["ConsecutiveFailures"] = _consecutiveFailures,
                ["IntervalMS"] = _options.IntervalMS
            };

            if (_lastException != null)
            {
                data["LastException"] = _lastException.Message;
            }

            return Task.FromResult(HealthCheckResult.Unhealthy(
                $"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)",
                _lastException,
                data
            ));
        }

        // Degraded: 1-2 consecutive failures but within time limit
        if (_consecutiveFailures > 0)
        {
            var data = new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun,
                ["ConsecutiveFailures"] = _consecutiveFailures
            };

            return Task.FromResult(HealthCheckResult.Degraded(
                $"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational",
                null,
                data
            ));
        }

        // Healthy
        return Task.FromResult(HealthCheckResult.Healthy(
            "ProfileWorker is running normally",
            new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun
            }
        ));
    }
}