Configure mass transit to send to only one consumer per instance

Question

I would like to use MassTransit in a way that would allow my application to have multiple consumers that are all consuming messages for the same type, but have it so that only one consumer ever receives each message. I do not care which one receives any particular message, only that it is handled once. Below is the code that I initially thought would work.

            services.AddScoped<S3ToArchiveConsumer<CaringoConfig, WasabiEastOneConfig>>();     
            services.AddScoped<S3ToArchiveConsumer<CaringoConfig, WasabiEastTwoConfig>>();           
            services.AddMassTransit(x =>
            {
                x.SetKebabCaseEndpointNameFormatter();
                x.UsingRabbitMq((context, cfg) =>
                {
                    CaringoShovelSettings caringoShovel = context.GetRequiredService<IOptions<CaringoShovelSettings>>().Value;
                    AppSettings appSettings = context.GetRequiredService<IOptions<AppSettings>>().Value;
                    cfg.Host(appSettings.RabbitMqHost, "/", h =>
                    {
                        h.Username(appSettings.RabbitMqUsername);
                        h.Password(appSettings.RabbitMqPassword);
                    });
                    if ((caringoShovel.WasabiEastOneShovel?.Enabled ?? false) || (caringoShovel.WasabiEastTwoShovel?.Enabled ?? false))
                    {
                        cfg.ReceiveEndpoint(nameof(MoveCaringoFileRequest), e =>
                        {
                            e.Durable = true;
                            e.Exclusive = false;
                            e.ExclusiveConsumer = false;
                            e.ThrowOnSkippedMessages();
                            if (caringoShovel.WasabiEastOneShovel?.Enabled ?? false)
                            {
                               
                                e.Consumer<S3ToArchiveConsumer<CaringoConfig, WasabiEastOneConfig>>(context, configure =>
                                 {
                                     configure.UseTimeout(x => x.Timeout = caringoShovel.WasabiEastOneShovel.Timeout);
                                     configure.UseConcurrentMessageLimit(caringoShovel.WasabiEastOneShovel.Concurrency);
                                 });
                            }
                            if (caringoShovel.WasabiEastTwoShovel != null && caringoShovel.WasabiEastTwoShovel.Enabled == true)
                            {
                                
                                e.Consumer<S3ToArchiveConsumer<CaringoConfig, WasabiEastTwoConfig>>(context, configure =>
                                 {
                                     configure.UseTimeout(x => x.Timeout = caringoShovel.WasabiEastTwoShovel.Timeout);
                                     configure.UseConcurrentMessageLimit(caringoShovel.WasabiEastTwoShovel.Concurrency);
                                 });
                            }
                        });
                    }
                    cfg.ConfigureEndpoints(context);
                });
            });
            services.AddMassTransitHostedService();

Answer 1

You didn't post the errors you are seeing, or what's coming out of the log, but there are quite a few ordering issues with the above configuration (which I've cleaned up below). Also, you should ensure that every receive endpoint is configured identically, to avoid skipped messages. Throwing on skipped messages could cause unrecognized messages to block the queue, which is bad, so having a single queue with multiple message types – some of which may not be consumed by consumers on that queue – can lead to issues with your configuration. MassTransit normally moves unrecognized messages to the _skipped queue to prevent these issues.

Anyway, here is the cleaned up configuration:

services.AddScoped<S3ToArchiveConsumer<CaringoConfig, WasabiEastOneConfig>>();
services.AddScoped<S3ToArchiveConsumer<CaringoConfig, WasabiEastTwoConfig>>();

services.AddMassTransit(x =>
{
    x.SetKebabCaseEndpointNameFormatter();

    x.UsingRabbitMq((context, cfg) =>
    {
        CaringoShovelSettings caringoShovel = context.GetRequiredService<IOptions<CaringoShovelSettings>>().Value;
        AppSettings appSettings = context.GetRequiredService<IOptions<AppSettings>>().Value;

        cfg.Host(appSettings.RabbitMqHost, "/", h =>
        {
            h.Username(appSettings.RabbitMqUsername);
            h.Password(appSettings.RabbitMqPassword);
        });

        if ((caringoShovel.WasabiEastOneShovel?.Enabled ?? false) || (caringoShovel.WasabiEastTwoShovel?.Enabled ?? false))
        {
            cfg.ReceiveEndpoint(nameof(MoveCaringoFileRequest), e =>
            {
                e.ThrowOnSkippedMessages();

                if (caringoShovel.WasabiEastOneShovel?.Enabled ?? false)
                {
                    e.ConcurrentMessageLimit = caringoShovel.WasabiEastOneShovel.Concurrency;
                    e.Consumer<S3ToArchiveConsumer<CaringoConfig, WasabiEastOneConfig>>(context, configure =>
                    {
                        configure.UseTimeout(x => x.Timeout = caringoShovel.WasabiEastOneShovel.Timeout);
                    });
                }
                if (caringoShovel.WasabiEastTwoShovel?.Enabled ?? false)
                {
                    // this may overwrite the value above if both are specified
                    e.ConcurrentMessageLimit = caringoShovel.WasabiEastTwoShovel.Concurrency;

                    e.Consumer<S3ToArchiveConsumer<CaringoConfig, WasabiEastTwoConfig>>(context, configure =>
                    {
                        configure.UseTimeout(x => x.Timeout = caringoShovel.WasabiEastTwoShovel.Timeout);
                    });
                }
            });
        }
    });
});
services.AddMassTransitHostedService();

The more I think about your question, I've come to suggest that this should be two completely separate receive endpoints, and you should publish the message using MassTransit, so that each queue receives a copy of it. And only one consumer on each receive endpoint.