From 8a05d8628520c871473ce8f64743c7dbd403a937 Mon Sep 17 00:00:00 2001 From: TekH Date: Mon, 13 Jul 2026 12:19:11 +0200 Subject: [PATCH] feat: Add health check monitoring for ProfileWorker ProfileWorker changes: - Implement IHealthCheck interface - Replace _workCount with DateTime-based tracking - Track _lastSuccessfulRun, _consecutiveFailures, _lastException - Graceful shutdown handling (OperationCanceledException) - Health states: Healthy, Degraded (1-2 failures), Unhealthy (3x interval) DependencyInjection changes: - Register ProfileWorker as singleton (for health check access) - Use factory pattern for IHostedService registration Program.cs changes: - Add health check service with ProfileWorker - Map /health endpoint (full JSON response with all checks) - Map /health/ready endpoint (filtered by 'ready' tag) - Custom JSON response writer with detailed metrics --- .../ProfileWorker/DependencyInjection.cs | 6 +- .../ProfileWorker/ProfileWorker.cs | 98 +++++++++++++++++-- ECMJobRunner.WebCron/Program.cs | 34 +++++++ 3 files changed, 128 insertions(+), 10 deletions(-) diff --git a/ECMJobRunner.WebCron/ProfileWorker/DependencyInjection.cs b/ECMJobRunner.WebCron/ProfileWorker/DependencyInjection.cs index ead8be1..c5e38ef 100644 --- a/ECMJobRunner.WebCron/ProfileWorker/DependencyInjection.cs +++ b/ECMJobRunner.WebCron/ProfileWorker/DependencyInjection.cs @@ -13,9 +13,13 @@ public static class DependencyInjection // Validate options at startup services.AddSingleton, ProfileWorkerOptionsValidator>(); - services.AddHostedService(); + // Register ProfileWorker as both HostedService and singleton (for health check access) + services.AddSingleton(); + services.AddHostedService(sp => sp.GetRequiredService()); + services.AddSingleton(); services.AddScoped(); + return services; } } diff --git a/ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs b/ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs index 982e9da..26d04d5 100644 --- a/ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs +++ b/ECMJobRunner.WebCron/ProfileWorker/ProfileWorker.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Options; namespace ECMJobRunner.WebCron.ProfileWorker; @@ -5,35 +6,114 @@ namespace ECMJobRunner.WebCron.ProfileWorker; public class ProfileWorker( ILogger Logger, IServiceScopeFactory ScopeFactory, - IOptions Options) : BackgroundService + IOptions Options) : BackgroundService, IHealthCheck { private readonly ProfileWorkerOptions _options = Options.Value; - private int _workCount = 0; + // Health check state + private DateTime _lastSuccessfulRun = DateTime.UtcNow; + private int _consecutiveFailures = 0; + private Exception? _lastException = null; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + Logger.LogInformation("ProfileWorker started"); + while (!stoppingToken.IsCancellationRequested) { - _workCount++; - if (Logger.IsEnabled(LogLevel.Information)) - { - Logger.LogInformation("Worker running {workCount} at: {time}", _workCount, DateTimeOffset.Now); - } - try { // Create a scope to resolve scoped services (ISQLExecutor used by MediatR pipeline) using var scope = ScopeFactory.CreateScope(); var work = scope.ServiceProvider.GetRequiredService(); await work.ExecuteAsync(stoppingToken); + + // Success - update health state + _lastSuccessfulRun = DateTime.UtcNow; + _consecutiveFailures = 0; + _lastException = null; + + if (Logger.IsEnabled(LogLevel.Debug)) + { + Logger.LogDebug("ProfileWorker sync completed successfully at {time}", _lastSuccessfulRun); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + // Graceful shutdown - app is stopping + Logger.LogInformation("ProfileWorker stopping due to cancellation request"); + break; } catch (Exception ex) { - Logger.LogError(ex, "An unexpected error occurred in ProfileWorker. Work count: {workCount}", _workCount); + // Unexpected error - track for health check + _consecutiveFailures++; + _lastException = ex; + + Logger.LogError(ex, + "An unexpected error occurred in ProfileWorker (consecutive failures: {failures})", + _consecutiveFailures); } await Task.Delay(_options.IntervalMS, stoppingToken); } + + Logger.LogInformation("ProfileWorker stopped"); + } + + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + var timeSinceLastSuccess = DateTime.UtcNow - _lastSuccessfulRun; + var maxAllowedDelay = TimeSpan.FromMilliseconds(_options.IntervalMS * 3); + + // Unhealthy: No successful run for 3x interval + if (timeSinceLastSuccess > maxAllowedDelay) + { + var data = new Dictionary + { + ["LastSuccessfulRun"] = _lastSuccessfulRun, + ["TimeSinceLastSuccess"] = timeSinceLastSuccess, + ["ConsecutiveFailures"] = _consecutiveFailures, + ["IntervalMS"] = _options.IntervalMS + }; + + if (_lastException != null) + { + data["LastException"] = _lastException.Message; + } + + return Task.FromResult(HealthCheckResult.Unhealthy( + $"ProfileWorker has not completed successfully for {timeSinceLastSuccess.TotalSeconds:F0} seconds ({_consecutiveFailures} consecutive failures)", + _lastException, + data + )); + } + + // Degraded: 1-2 consecutive failures but within time limit + if (_consecutiveFailures > 0) + { + var data = new Dictionary + { + ["LastSuccessfulRun"] = _lastSuccessfulRun, + ["ConsecutiveFailures"] = _consecutiveFailures + }; + + return Task.FromResult(HealthCheckResult.Degraded( + $"ProfileWorker has {_consecutiveFailures} consecutive failure(s) but still operational", + null, + data + )); + } + + // Healthy + return Task.FromResult(HealthCheckResult.Healthy( + "ProfileWorker is running normally", + new Dictionary + { + ["LastSuccessfulRun"] = _lastSuccessfulRun + } + )); } } diff --git a/ECMJobRunner.WebCron/Program.cs b/ECMJobRunner.WebCron/Program.cs index 26ae8c0..80e0ce8 100644 --- a/ECMJobRunner.WebCron/Program.cs +++ b/ECMJobRunner.WebCron/Program.cs @@ -101,6 +101,10 @@ builder.Services.AddHangfireServer(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); +// Add health checks +builder.Services.AddHealthChecks() + .AddCheck("profile-worker", tags: new[] { "ready", "worker" }); + // Add Serilog.UI with SQLite provider - use same path from configuration var serilogUiLogDirectory = builder.Configuration.GetValue("Application:LogDirectory") ?? throw new InvalidOperationException("Application:LogDirectory not found in configuration."); @@ -139,6 +143,36 @@ app.UseSerilogUi(); app.MapControllers(); +// Map health check endpoints +app.MapHealthChecks("/health", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions +{ + Predicate = _ => true, + ResponseWriter = async (context, report) => + { + context.Response.ContentType = "application/json"; + var result = System.Text.Json.JsonSerializer.Serialize(new + { + status = report.Status.ToString(), + timestamp = DateTime.UtcNow, + checks = report.Entries.Select(e => new + { + name = e.Key, + status = e.Value.Status.ToString(), + description = e.Value.Description, + duration = e.Value.Duration.TotalMilliseconds, + exception = e.Value.Exception?.Message, + data = e.Value.Data + }) + }); + await context.Response.WriteAsync(result); + } +}); + +app.MapHealthChecks("/health/ready", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions +{ + Predicate = check => check.Tags.Contains("ready") +}); + // Redirect root path to Hangfire dashboard app.MapGet("/", () => Results.Redirect("/hangfire"));