feat(infrastructure): publish worker status to health checks
ConsumerInitWorker marks ConsumerInitHealthCheck as unhealthy when the consumer pool fails to start, so callers receive HTTP 503 instead of a silently degraded service. EmailSyncWorker now reports its startup interval and the result of every sync cycle. SyncAccountAsync returns whether the account failed, letting the worker count errors per cycle and expose them through EmailSyncWorkerHealthCheck.
This commit is contained in:
@@ -11,7 +11,11 @@ namespace DigitalData.MessagingService.Infrastructure.Services.Background;
|
||||
/// Leverages a push-based, event-driven RabbitMQ consumer to eliminate polling overhead.
|
||||
/// Email account configuration is resolved exclusively from application settings; no database access is performed.
|
||||
/// </summary>
|
||||
public class ConsumerInitWorker(SendingEmailConsumerPool ConsumerPool, ILogger<ConsumerInitWorker> Logger, IOptions<RabbitMqConfiguration> RabbitMqOptions) : BackgroundService
|
||||
public class ConsumerInitWorker(
|
||||
SendingEmailConsumerPool ConsumerPool,
|
||||
ILogger<ConsumerInitWorker> Logger,
|
||||
IOptions<RabbitMqConfiguration> RabbitMqOptions,
|
||||
ConsumerInitHealthCheck HealthCheck) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
@@ -21,10 +25,12 @@ public class ConsumerInitWorker(SendingEmailConsumerPool ConsumerPool, ILogger<C
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Log and continue — a RabbitMQ connection failure must not bring down the entire host.
|
||||
// Log and mark the health check as unhealthy — callers will receive HTTP 503.
|
||||
// The API remains operational; only outgoing email delivery via the consumer pool is affected.
|
||||
Logger.LogError(ex, "Failed to initialize SendingEmailConsumerPool. Outgoing email consumers are not running.");
|
||||
|
||||
HealthCheck.MarkUnhealthy(ex.Message);
|
||||
|
||||
if (!RabbitMqOptions.Value.ContinueOnInitFailure)
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ using Microsoft.Extensions.Options;
|
||||
|
||||
namespace DigitalData.MessagingService.Infrastructure.Services.Background;
|
||||
|
||||
public class EmailSyncWorker(IOptions<EmailAccountsOptions> Options, IServiceProvider Provider, ILogger<EmailSyncWorker> Logger, IMemoryCache Cache) : BackgroundService, IEmailSyncService
|
||||
public class EmailSyncWorker(IOptions<EmailAccountsOptions> Options, IServiceProvider Provider, ILogger<EmailSyncWorker> Logger, IMemoryCache Cache, EmailSyncWorkerHealthCheck HealthCheck) : BackgroundService, IEmailSyncService
|
||||
{
|
||||
private static string ForcedSyncDateCacheKey => $"{nameof(EmailSyncWorker)}_TriggerSync";
|
||||
|
||||
@@ -44,6 +44,7 @@ public class EmailSyncWorker(IOptions<EmailAccountsOptions> Options, IServicePro
|
||||
await UpsertSeedEmailAccount(stoppingToken);
|
||||
|
||||
var interval = TimeSpan.FromSeconds(Options.Value.SyncIntervalSeconds);
|
||||
HealthCheck.OnWorkerStarted(interval);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -53,11 +54,15 @@ public class EmailSyncWorker(IOptions<EmailAccountsOptions> Options, IServicePro
|
||||
var imapService = scope.ServiceProvider.GetRequiredService<IImapEmailService>();
|
||||
var pop3Service = scope.ServiceProvider.GetRequiredService<IPop3EmailService>();
|
||||
|
||||
int cycleErrorCount = 0;
|
||||
foreach (var account in await emailAccountRepo.GetAllAsync(stoppingToken))
|
||||
{
|
||||
await SyncAccountAsync(account, imapService, pop3Service, stoppingToken);
|
||||
var hadError = await SyncAccountAsync(account, imapService, pop3Service, stoppingToken);
|
||||
if (hadError) cycleErrorCount++;
|
||||
}
|
||||
|
||||
HealthCheck.OnCycleCompleted(cycleErrorCount);
|
||||
|
||||
// Reset trigger before waiting so any TriggerSync() call during the delay is caught
|
||||
_syncTrigger = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
@@ -70,14 +75,15 @@ public class EmailSyncWorker(IOptions<EmailAccountsOptions> Options, IServicePro
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SyncAccountAsync(
|
||||
/// <returns><c>true</c> if the sync failed for this account; otherwise <c>false</c>.</returns>
|
||||
private async Task<bool> SyncAccountAsync(
|
||||
EmailAccount account,
|
||||
IImapEmailService imapService,
|
||||
IPop3EmailService pop3Service,
|
||||
CancellationToken stoppingToken)
|
||||
{
|
||||
if (account.IncomingProtocol == IncomingProtocol.None)
|
||||
return;
|
||||
return false;
|
||||
|
||||
Logger.LogDebug(
|
||||
"Email sync started. Account={Username}, Protocol={Protocol}.",
|
||||
@@ -100,12 +106,16 @@ public class EmailSyncWorker(IOptions<EmailAccountsOptions> Options, IServicePro
|
||||
Logger.LogDebug(
|
||||
"Email sync completed. Account={Username}, Protocol={Protocol}, Processed={Processed}, Failed={Failed}.",
|
||||
account.Username, account.IncomingProtocol, result.ProcessedCount, result.FailedCount);
|
||||
|
||||
return false;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logger.LogError(ex,
|
||||
"Email sync failed. Account={Username}, Protocol={Protocol}.",
|
||||
account.Username, account.IncomingProtocol);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user