Introduce RabbitMQ consumer pool for parallel processing
Enhanced RabbitMQ email processing by introducing a `SendingEmailConsumerPool` to enable the competing consumers pattern. Each consumer operates on its own channel, improving scalability and thread safety. - Added `SendingEmailConsumerPool` to manage multiple consumers. - Updated `DependencyInjection` to register the consumer pool. - Refactored `SendingEmailConsumer` for better logging and error handling. - Updated `AsyncInitWorker` to initialize the consumer pool. - Added `ConsumerConcurrency` to RabbitMQ configuration. - Improved error handling in `LimilabsEmailService` with detailed SMTP error messages.
This commit is contained in:
@@ -4,20 +4,19 @@ using DigitalData.MessagingService.Application.Common.Interfaces;
|
||||
using DigitalData.MessagingService.Abstraction;
|
||||
using DigitalData.MessagingService.RabbitMQ;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using RabbitMQ.Client;
|
||||
using RabbitMQ.Client.Events;
|
||||
|
||||
namespace DigitalData.MessagingService.Infrastructure.Queue;
|
||||
|
||||
/// <summary>
|
||||
/// RabbitMQ-based email queue implementation for outgoing emails.
|
||||
/// Provides message persistence, scalability, and reliability.
|
||||
/// Uses Lazy<T> initialization pattern to avoid blocking constructor.
|
||||
/// A single RabbitMQ consumer that processes one email message at a time on its own dedicated channel.
|
||||
/// Multiple instances run in parallel via <see cref="SendingEmailConsumerPool"/> (competing consumers pattern).
|
||||
/// Each instance owns exactly one channel — channels are not thread-safe and must not be shared.
|
||||
/// </summary>
|
||||
public sealed class SendingEmailConsumer : IAsyncDisposable
|
||||
{
|
||||
private readonly RabbitMqConfiguration _config;
|
||||
private readonly string _queueName;
|
||||
|
||||
private readonly Lazy<Task<IChannel>> _lazyChannel;
|
||||
|
||||
@@ -25,15 +24,27 @@ public sealed class SendingEmailConsumer : IAsyncDisposable
|
||||
|
||||
private readonly ILogger<SendingEmailConsumer>? _logger;
|
||||
|
||||
public SendingEmailConsumer(IOptions<RabbitMqConfiguration> config, IEmailService EmailService, RabbitMqConnectionFactory CnnFactory, ILogger<SendingEmailConsumer>? logger = null)
|
||||
/// <summary>
|
||||
/// Transient identifier assigned to this consumer instance at runtime.
|
||||
/// A new value is generated each time the application starts or a new consumer is created.
|
||||
/// Use this to correlate log entries belonging to the same consumer session across competing instances.
|
||||
/// </summary>
|
||||
public Guid RuntimeId { get; } = Guid.NewGuid();
|
||||
|
||||
public SendingEmailConsumer(string queueName, IEmailService emailService, RabbitMqConnectionFactory cnnFactory, ILogger<SendingEmailConsumer>? logger = null)
|
||||
{
|
||||
_logger = logger;
|
||||
_config = config.Value;
|
||||
_queueName = queueName;
|
||||
|
||||
_lazyChannel = new(CnnFactory.CreateChannelAsync);
|
||||
_lazyInit = new(async () => {
|
||||
_lazyChannel = new(cnnFactory.CreateChannelAsync);
|
||||
_lazyInit = new(async () =>
|
||||
{
|
||||
var channel = await _lazyChannel.Value;
|
||||
|
||||
// prefetchCount=1 ensures this consumer processes one message at a time before acking.
|
||||
// Parallelism comes from running multiple consumer instances, not from within a single channel.
|
||||
await channel.BasicQosAsync(prefetchSize: 0, prefetchCount: 1, global: false);
|
||||
|
||||
var consumer = new AsyncEventingBasicConsumer(channel);
|
||||
|
||||
consumer.ReceivedAsync += async (sender, args) =>
|
||||
@@ -47,10 +58,14 @@ public sealed class SendingEmailConsumer : IAsyncDisposable
|
||||
if (oMailEvent is not null)
|
||||
{
|
||||
// Send email via SMTP (SMTP config is injected in IEmailService via IOptions)
|
||||
await EmailService.SendEmailAsync(oMailEvent.Mail);
|
||||
await emailService.SendEmailAsync(oMailEvent.Mail, args.CancellationToken);
|
||||
|
||||
// Acknowledge message after successful processing
|
||||
await channel.BasicAckAsync(args.DeliveryTag, false, args.CancellationToken);
|
||||
|
||||
logger?.LogDebug(
|
||||
"Email successfully sent and acknowledged. RuntimeId={RuntimeId}, Queue={QueueName}, DeliveryTag={DeliveryTag}, To={Recipients}, Subject={Subject}, EventId={EventId}",
|
||||
RuntimeId, _queueName, args.DeliveryTag, oMailEvent.Mail.Recipients, oMailEvent.Mail.Subject, oMailEvent.Id);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -89,19 +104,18 @@ public sealed class SendingEmailConsumer : IAsyncDisposable
|
||||
|
||||
// Start consuming messages (event-driven, non-blocking)
|
||||
await channel.BasicConsumeAsync(
|
||||
queue: _config.QueueName,
|
||||
queue: _queueName,
|
||||
autoAck: false,
|
||||
consumer: consumer,
|
||||
cancellationToken: CnnFactory.CancellationToken);
|
||||
cancellationToken: cnnFactory.CancellationToken);
|
||||
|
||||
logger?.LogInformation("RabbitMQ consumer started for queue: {QueueName}", _config.QueueName);
|
||||
logger?.LogInformation("RabbitMQ consumer started for queue: {QueueName}", _queueName);
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initialize RabbitMQ connection, channel, exchanges, and queues asynchronously.
|
||||
/// Start event-driven consumer that processes messages as they arrive
|
||||
/// Called lazily on first use via EnsureInitializedAsync.
|
||||
/// Starts the consumer: opens a channel, sets QoS, and registers the event handler.
|
||||
/// Called by <see cref="SendingEmailConsumerPool.InitAsync"/>.
|
||||
/// </summary>
|
||||
public async Task InitAsync()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user