ASP.NET Core 3.1 Hosted Service runs multiple times for a given schedule (its supposed to run once a day)

Question

I've this hosted service job in asp.net core 3.1 application running in the background. In development everything works fine but when I deploy this to production whichever job is inherited from this abstract class will run the ExecuteOnStartAsync() method multiple times. In development when I test the DummyTestJob class. It runs perfectly fine and is called once every 10 seconds. However when I deploy this to production with a job that's supposed to send 1 email once at 2:00am, in the am I will have anywhere between 5-13 emails and this number is arbitrary too. This is deployed on an AWS EC2 Windows machine and I'm not sure if it has anything to do with the number of cpu cores and so on however my local development workstation has 8 cores and it still runs once. So I'm not sure what else is going on. Have you experienced anything like this before with asp.net core hosted service?

public abstract class ScheduledBackgroundWorker<T> : BaseBackgroundWorker<T> where T : class
{
    private CrontabSchedule schedule;
    private DateTime nextRun;
    protected string JobName => this.GetType().Name;
    protected abstract string JobDescription { get; }
    protected abstract string Schedule { get; }

    protected ScheduledBackgroundWorker(IServiceProvider services,
       ILogger<T> logger) : base(services, logger)
    {
        schedule = CrontabSchedule.Parse(Schedule, new CrontabSchedule.ParseOptions
        {
            IncludingSeconds = true
        });
        nextRun = schedule.GetNextOccurrence(DateTime.Now);
    }

    protected abstract Task ExecuteOnStartAsync(CancellationToken cancellationToken);
    protected abstract Task ExecuteOnStopAsync(CancellationToken cancellationToken);

    protected override async Task ExecuteStartAsync(CancellationToken cancellationToken)
    {
        logger.LogInformation("{jobName} Schedule {schedule} was scheduled and next run will be at {nextRun} - {now}", JobName, Schedule, nextRun.ToString("MM/dd/yyyy hh:mm:ss t"), DateTime.Now);
        using var scope = services.CreateScope();
        var cache =
            scope.ServiceProvider
                .GetRequiredService<IDistributedCache>();
        do
        {
            var hasRun = await cache.GetAsync(JobName);
            if (hasRun == null)
            {
                var now = DateTime.Now;
                if (now > nextRun)
                {
                    logger.LogInformation("{jobName} Schedule {schedule} is running - {now}", JobName, Schedule, DateTime.Now);
                    await ExecuteOnStartAsync(cancellationToken);
                    nextRun = schedule.GetNextOccurrence(DateTime.Now);

                    var currentTime = nextRun.ToString();
                    byte[] encodedCurrentTime = Encoding.UTF8.GetBytes(currentTime);
                    var options = new DistributedCacheEntryOptions().SetAbsoluteExpiration(TimeSpan.FromSeconds(60));
                    await cache.SetAsync(JobName, encodedCurrentTime, options, cancellationToken);

                    logger.LogInformation("{jobName} Schedule {schedule} has finished and next run will be at {nextRun} - {now}", JobName, Schedule, nextRun.ToString("MM/dd/yyyy hh:mm:ss t"), DateTime.Now);
                }
            }
            await Task.Delay(5000, cancellationToken); //5 seconds delay
        }
        while (!cancellationToken.IsCancellationRequested);

        await Task.CompletedTask;
    }

    protected override async Task ExecuteStopAsync(CancellationToken cancellationToken)
    {
        await ExecuteOnStopAsync(cancellationToken);
        logger.LogInformation("{jobName} Schedule {schedule} was stopped - {now}", JobName, Schedule, DateTime.Now);
        await Task.CompletedTask;
    }

    public override void Dispose()
    {
    }
}

Even with saving the running state in cache it will do the same thing as without cache. I'm using this library called NCrontab to set the cron schedule. Not sure if it has anything to do with it. But again it works in development, fails in production.

public class DummyTestJob : ScheduledBackgroundWorker<DummyTestJob>
{
public DummyTestJob(IServiceProvider services,
    ILogger<DummyTestJob> logger) : base(services, logger)
{

}

protected override string JobDescription => "Dummy Test";

protected override string Schedule => "10 * * * * *";

protected override async Task ExecuteOnStartAsync(CancellationToken cancellationToken)
{
    using var scope = services.CreateScope();
    var apiClient =
        scope.ServiceProvider
            .GetRequiredService<IApiClient>();

    var runtimeEnvironment = scope.ServiceProvider.GetRequiredService<IRuntimeEnvironment>();

    try
    {
        var page = 1;
        var size = 100;
        var loop = true;
        List<User> users = new List<User>();
        do
        {
            try
            {
                var response = await apiClient.GetAsync($"users?page={page}&size={size}", null, null);

                if (response.Count > 0)
                {
                    users.AddRange(response.Data);
                    page++;
                }
                else
                {
                    loop = false;
                }
            }
            catch (Exception ex)
            {
                logger.LogError(ex, "Failure getting users for page {page}", page);
                loop = false;
            }

        } while (loop);

        List<string> userNames = new List<string>();

        foreach (var user in Users)
        {
            if (user.IsActive == 1)
            {
                userNames.Add(user.Name);
            }
        }

        users.Clear();
        users = null;

        foreach (var userName in userNames)
        {
            System.Console.WriteLine($"Hello {userName}...\n");
        }
    }
    catch (OperationCanceledException ex)
    {
        logger.LogError(ex, "{JobName} execution canceled - {now}", JobName, DateTime.Now);
    }

    await Task.CompletedTask;
}

protected override async Task ExecuteOnStopAsync(CancellationToken cancellationToken)
{
    await Task.CompletedTask;
}
}


public abstract class BaseBackgroundWorker<T> : BackgroundService where T : class
{
    protected readonly IServiceProvider services;
    protected readonly ILogger<T> logger;

    // inject a logger
    protected BaseBackgroundWorker(IServiceProvider services,
        ILogger<T> logger)
    {
        this.services = services;
        this.logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken cancellationToken)
    {
        await ExecuteStartAsync(cancellationToken);
        await Task.CompletedTask;
    }

    protected abstract Task ExecuteStartAsync(CancellationToken cancellationToken);
    protected abstract Task ExecuteStopAsync(CancellationToken cancellationToken);

    public override async Task StopAsync(CancellationToken cancellationToken)
    {
        await ExecuteStopAsync(cancellationToken);
        await base.StopAsync(cancellationToken);
        await Task.CompletedTask;
    }
}

Answer 1

It was because in IIS I had set the Max Worker Process to 5 for the API. Resetting it back to 1 fixed the issue