Introduce RabbitMQ consumer pool for parallel processing

Enhanced RabbitMQ email processing by introducing a `SendingEmailConsumerPool` to enable the competing consumers pattern. Each consumer operates on its own channel, improving scalability and thread safety.

- Added `SendingEmailConsumerPool` to manage multiple consumers.
- Updated `DependencyInjection` to register the consumer pool.
- Refactored `SendingEmailConsumer` for better logging and error handling.
- Updated `AsyncInitWorker` to initialize the consumer pool.
- Added `ConsumerConcurrency` to RabbitMQ configuration.
- Improved error handling in `LimilabsEmailService` with detailed SMTP error messages.
This commit is contained in:
2026-08-05 15:34:33 +02:00
parent bd31bfe528
commit fa9b4973b9
6 changed files with 114 additions and 28 deletions

View File

@@ -4,20 +4,19 @@ using DigitalData.MessagingService.Application.Common.Interfaces;
using DigitalData.MessagingService.Abstraction;
using DigitalData.MessagingService.RabbitMQ;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using RabbitMQ.Client;
using RabbitMQ.Client.Events;
namespace DigitalData.MessagingService.Infrastructure.Queue;
/// <summary>
/// RabbitMQ-based email queue implementation for outgoing emails.
/// Provides message persistence, scalability, and reliability.
/// Uses Lazy<T> initialization pattern to avoid blocking constructor.
/// A single RabbitMQ consumer that processes one email message at a time on its own dedicated channel.
/// Multiple instances run in parallel via <see cref="SendingEmailConsumerPool"/> (competing consumers pattern).
/// Each instance owns exactly one channel — channels are not thread-safe and must not be shared.
/// </summary>
public sealed class SendingEmailConsumer : IAsyncDisposable
{
private readonly RabbitMqConfiguration _config;
private readonly string _queueName;
private readonly Lazy<Task<IChannel>> _lazyChannel;
@@ -25,15 +24,27 @@ public sealed class SendingEmailConsumer : IAsyncDisposable
private readonly ILogger<SendingEmailConsumer>? _logger;
public SendingEmailConsumer(IOptions<RabbitMqConfiguration> config, IEmailService EmailService, RabbitMqConnectionFactory CnnFactory, ILogger<SendingEmailConsumer>? logger = null)
/// <summary>
/// Transient identifier assigned to this consumer instance at runtime.
/// A new value is generated each time the application starts or a new consumer is created.
/// Use this to correlate log entries belonging to the same consumer session across competing instances.
/// </summary>
public Guid RuntimeId { get; } = Guid.NewGuid();
public SendingEmailConsumer(string queueName, IEmailService emailService, RabbitMqConnectionFactory cnnFactory, ILogger<SendingEmailConsumer>? logger = null)
{
_logger = logger;
_config = config.Value;
_queueName = queueName;
_lazyChannel = new(CnnFactory.CreateChannelAsync);
_lazyInit = new(async () => {
_lazyChannel = new(cnnFactory.CreateChannelAsync);
_lazyInit = new(async () =>
{
var channel = await _lazyChannel.Value;
// prefetchCount=1 ensures this consumer processes one message at a time before acking.
// Parallelism comes from running multiple consumer instances, not from within a single channel.
await channel.BasicQosAsync(prefetchSize: 0, prefetchCount: 1, global: false);
var consumer = new AsyncEventingBasicConsumer(channel);
consumer.ReceivedAsync += async (sender, args) =>
@@ -47,10 +58,14 @@ public sealed class SendingEmailConsumer : IAsyncDisposable
if (oMailEvent is not null)
{
// Send email via SMTP (SMTP config is injected in IEmailService via IOptions)
await EmailService.SendEmailAsync(oMailEvent.Mail);
await emailService.SendEmailAsync(oMailEvent.Mail, args.CancellationToken);
// Acknowledge message after successful processing
await channel.BasicAckAsync(args.DeliveryTag, false, args.CancellationToken);
logger?.LogDebug(
"Email successfully sent and acknowledged. RuntimeId={RuntimeId}, Queue={QueueName}, DeliveryTag={DeliveryTag}, To={Recipients}, Subject={Subject}, EventId={EventId}",
RuntimeId, _queueName, args.DeliveryTag, oMailEvent.Mail.Recipients, oMailEvent.Mail.Subject, oMailEvent.Id);
}
else
{
@@ -89,19 +104,18 @@ public sealed class SendingEmailConsumer : IAsyncDisposable
// Start consuming messages (event-driven, non-blocking)
await channel.BasicConsumeAsync(
queue: _config.QueueName,
queue: _queueName,
autoAck: false,
consumer: consumer,
cancellationToken: CnnFactory.CancellationToken);
cancellationToken: cnnFactory.CancellationToken);
logger?.LogInformation("RabbitMQ consumer started for queue: {QueueName}", _config.QueueName);
logger?.LogInformation("RabbitMQ consumer started for queue: {QueueName}", _queueName);
});
}
/// <summary>
/// Initialize RabbitMQ connection, channel, exchanges, and queues asynchronously.
/// Start event-driven consumer that processes messages as they arrive
/// Called lazily on first use via EnsureInitializedAsync.
/// Starts the consumer: opens a channel, sets QoS, and registers the event handler.
/// Called by <see cref="SendingEmailConsumerPool.InitAsync"/>.
/// </summary>
public async Task InitAsync()
{