ECMJobRunner/ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs

using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Options;

namespace ECMJobRunner.WebCron.ProfileWorker;

/// <summary>
/// Background service that continuously synchronizes active profiles with Hangfire recurring jobs.
/// Implements health check monitoring to track service status and failure conditions.
/// </summary>
/// <param name="Logger">Logger for diagnostic output.</param>
/// <param name="ScopeFactory">Factory for creating service scopes (required for scoped service resolution).</param>
/// <param name="Options">Configuration options for interval timing.</param>
/// <remarks>
/// This service runs on a configurable interval (default 1 second) and performs the following:
/// <list type="bullet">
/// <item><description>Fetches active profiles from the database</description></item>
/// <item><description>Synchronizes profiles with Hangfire recurring jobs</description></item>
/// <item><description>Tracks health status based on success/failure patterns</description></item>
/// <item><description>Gracefully handles cancellation during application shutdown</description></item>
/// </list>
/// Health states (with adaptive thresholds):
/// <list type="bullet">
/// <item><description><b>Degraded (Initializing)</b>: Service starting up, waiting for first successful run</description></item>
/// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
/// <item><description><b>Degraded</b>: 1+ consecutive failures but within time threshold</description></item>
/// <item><description><b>Unhealthy</b>: No success within adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
/// </list>
/// The adaptive multiplier ensures fast problem detection when using longer intervals (e.g., 60s interval = 90s timeout)
/// while maintaining tolerance for network jitter on short intervals (e.g., 1s interval = 3s timeout).
/// </remarks>
public class ProfileWorker(
    ILogger<ProfileWorker> Logger, 
    IServiceScopeFactory ScopeFactory,
    IOptions<ProfileWorkerOptions> Options) : BackgroundService, IHealthCheck
{
    private readonly ProfileWorkerOptions _options = Options.Value;

    // Health check state
    private DateTime? _lastSuccessfulRun = null; // null = not run yet
    private int _consecutiveFailures = 0;
    private Exception? _lastException = null;

    /// <summary>
    /// Background service execution loop that synchronizes profiles on a configurable interval.
    /// </summary>
    /// <param name="stoppingToken">Cancellation token for graceful shutdown.</param>
    /// <returns>A task representing the background execution.</returns>
    /// <remarks>
    /// The loop continues until:
    /// <list type="bullet">
    /// <item><description>Application shutdown is requested (via stoppingToken)</description></item>
    /// <item><description>An unhandled exception causes service failure</description></item>
    /// </list>
    /// Uses scoped services for each iteration to ensure proper lifetime management.
    /// </remarks>
    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        Logger.LogInformation("ProfileWorker started");

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                // Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline)
                using var scope = ScopeFactory.CreateScope();
                var work = scope.ServiceProvider.GetRequiredService<ProfileWork>();
                await work.ExecuteAsync(stoppingToken);

                // Success - update health state
                _lastSuccessfulRun = DateTime.UtcNow;
                _consecutiveFailures = 0;
                _lastException = null;

                if (Logger.IsEnabled(LogLevel.Debug))
                {
                    Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun);
                }
            }
            catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
            {
                // Graceful shutdown - app is stopping
                Logger.LogInformation("ProfileWorker stopping due to cancellation request");
                break;
            }
            catch (Exception ex)
            {
                // Unexpected error - track for health check
                _consecutiveFailures++;
                _lastException = ex;

                Logger.LogError(ex, 
                    "An unexpected error occurred in ProfileWorker (consecutive failures: {failures})", 
                    _consecutiveFailures);
            }

            await Task.Delay(_options.IntervalMS, stoppingToken);
        }

        Logger.LogInformation("ProfileWorker stopped");
    }

    /// <summary>
    /// Performs a health check by evaluating recent success/failure patterns.
    /// </summary>
    /// <param name="context">Health check context (unused).</param>
    /// <param name="cancellationToken">Cancellation token.</param>
    /// <returns>
    /// A <see cref="HealthCheckResult"/> indicating the current health status:
    /// <list type="bullet">
    /// <item><description><see cref="HealthStatus.Healthy"/> - Service running normally</description></item>
    /// <item><description><see cref="HealthStatus.Degraded"/> - Recent failures but still operational, or service initializing</description></item>
    /// <item><description><see cref="HealthStatus.Unhealthy"/> - No successful run for extended period</description></item>
    /// </list>
    /// </returns>
    /// <remarks>
    /// Health determination logic:
    /// <list type="number">
    /// <item><description><b>Degraded (Initializing)</b>: Service has not completed its first successful run yet</description></item>
    /// <item><description><b>Unhealthy</b>: Time since last success exceeds adaptive threshold (3x interval for fast intervals &lt;10s, 1.5x for slower intervals)</description></item>
    /// <item><description><b>Degraded</b>: 1+ consecutive failures within time threshold</description></item>
    /// <item><description><b>Healthy</b>: Recent successful run with no failures</description></item>
    /// </list>
    /// Adaptive multiplier ensures fast problem detection for longer intervals while maintaining tolerance for short intervals.
    /// </remarks>
    public Task<HealthCheckResult> CheckHealthAsync(
        HealthCheckContext context, 
        CancellationToken cancellationToken = default)
    {
        // Service hasn't completed its first successful run yet
        if (_lastSuccessfulRun == null)
        {
            var data = new Dictionary<string, object>
            {
                ["Status"] = "Initializing",
                ["ConsecutiveFailures"] = _consecutiveFailures,
                ["IntervalMS"] = _options.IntervalMS
            };

            if (_lastException != null)
            {
                data["LastException"] = _lastException.Message;
            }

            // Degraded during initialization (not unhealthy, service is starting up)
            return Task.FromResult(HealthCheckResult.Degraded(
                _consecutiveFailures > 0 
                    ? $"ProfileWorker initializing with {_consecutiveFailures} failure(s)"
                    : "ProfileWorker is initializing, waiting for first successful run",
                _lastException,
                data
            ));
        }

        var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun.Value;
        
        // Adaptive multiplier: 3x for fast intervals (<10s), 1.5x for slower intervals
        // This ensures faster problem detection when using longer intervals (e.g., 60s)
        var multiplier = _options.IntervalMS < 10000 ? 3.0 : 1.5;
        var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * multiplier);

        // Unhealthy: No successful run within adaptive threshold
        if (timeSinceLastSuccess > maxAllowedDelay)
        {
            var data = new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
                ["TimeSinceLastSuccess"] = timeSinceLastSuccess,
                ["ConsecutiveFailures"] = _consecutiveFailures,
                ["IntervalMS"] = _options.IntervalMS,
                ["Multiplier"] = multiplier,
                ["MaxAllowedDelay"] = maxAllowedDelay
            };

            if (_lastException != null)
            {
                data["LastException"] = _lastException.Message;
            }

            return Task.FromResult(HealthCheckResult.Unhealthy(
                $"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)",
                _lastException,
                data
            ));
        }

        // Degraded: 1+ consecutive failures but within time limit
        if (_consecutiveFailures > 0)
        {
            var data = new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun.Value,
                ["ConsecutiveFailures"] = _consecutiveFailures
            };

            return Task.FromResult(HealthCheckResult.Degraded(
                $"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational",
                null,
                data
            ));
        }

        // Healthy
        return Task.FromResult(HealthCheckResult.Healthy(
            "ProfileWorker is running normally",
            new Dictionary<string, object>
            {
                ["LastSuccessfulRun"] = _lastSuccessfulRun.Value
            }
        ));
    }
}