stackexchange redis timeout exception cause website to timeout

Question

I'm using StackExchange NuGet in my ASP.NET MVC website with .NET Framework and when we have some a little more traffic the website cannot load and the request get stuck in IIS and i seee the request inside the Worker Processes in IIS and here is my connection helper for redis

public class RedisCacheHelper
    {
        public static T Get<T>(string cacheKey) where T : class
        {
            
                var cache = RedisConnectorHelper.Connection.GetDatabase();
                var itemValue = cache.StringGetAsync(cacheKey).Result;
                T item = null;
                if (!string.IsNullOrEmpty(itemValue))
                    item = JsonConvert.DeserializeObject<T>(itemValue);
                //T item = cache.StringGet(cacheKey) as T;
                return item;
            
        }
        public static void Set(string cacheKey, object ObjValue, int cacheMinutes = 60)
        {
            
                var cache = RedisConnectorHelper.Connection.GetDatabase();
                if (!string.IsNullOrEmpty(cacheKey) && ObjValue != null)
                {
                    var Expiration = TimeSpan.FromMinutes(cacheMinutes);
                    var redisVal = JsonConvert.SerializeObject(ObjValue);
                    cache.StringSetAsync(cacheKey, redisVal, Expiration, When.NotExists, CommandFlags.FireAndForget);
                }
            
        }
        public static void ClearRedisCache(string Key = null)
        {
            if (!string.IsNullOrEmpty(Key))
            {
                var cache = RedisConnectorHelper.Connection.GetDatabase();
                var endpoints = RedisConnectorHelper.Connection.GetEndPoints();
                var server = RedisConnectorHelper.Connection.GetServer(endpoints[0]);
                var keys = server.Keys();
                var _ObjKey = keys.Where(k => k.ToString().ToLower().Equals(Key.ToLower())).FirstOrDefault();
                if (!string.IsNullOrEmpty(_ObjKey))
                    cache.KeyDelete(_ObjKey);
            }
            else
            {
                var endpoints = RedisConnectorHelper.Connection.GetEndPoints();
                var server = RedisConnectorHelper.Connection.GetServer(endpoints[0]);
                server.FlushAllDatabases();
            }
        }

    }
    public class RedisConnectorHelper
    {
        static RedisConnectorHelper()
        {

            RedisConnectorHelper.lazyConnection = new Lazy<ConnectionMultiplexer>(() =>
            {
                return ConnectionMultiplexer.Connect("SomeConnection");
            });
        }

        private static Lazy<ConnectionMultiplexer> lazyConnection;

        public static ConnectionMultiplexer Connection
        {
            get
            {
                return lazyConnection.Value;
            }
        }
    }

and when checking the Event Viewer from the server i got the following

Event code: 3005 
Event message: An unhandled exception has occurred. 
Event time: 12/3/2022 4:53:53 AM 
Event time (UTC): 12/3/2022 2:53:53 AM 
Event ID: d5f0dd48b0554c508c5111b92599081e 
Event sequence: 56886 
Event occurrence: 5 
Event detail code: 0 
 
Application information: 
    Application domain: /LM/W3SVC/3/ROOT-1-133144634776972114 
    Trust level: Full 
    Application Virtual Path: / 
    Application Path: Some Path
    Machine name: EC2AMAZ-VG4448R 
 
Process information: 
    Process ID: 8412 
    Process name: w3wp.exe 
    Account name: EC2AMAZ-VG4448R\MyASPMVC_web 
 
Exception information: 
    Exception type: RedisTimeoutException 
    Exception message: Timeout performing SETEX (5000ms), next: GET AvailableResources, inst: 1345, qu: 0, qs: 852, aw: False, rs: ReadAsync, ws: Idle, in: 0, in-pipe: 0, out-pipe: 0, serverEndpoint: MyASPMVC-redis.gfqxrw.0001.use1.cache.amazonaws.com:6379, mc: 1/1/0, mgr: 10 of 10 available, clientName: EC2AMAZ-VG4448R, IOCP: (Busy=0,Free=1000,Min=16,Max=1000), WORKER: (Busy=647,Free=32120,Min=16,Max=32767), v: 2.2.50.36290 (Please take a look at this article for some common client-side issues that can cause timeouts: https://stackexchange.github.io/StackExchange.Redis/Timeouts)
   at StackExchange.Redis.ConnectionMultiplexer.ExecuteSyncImpl[T](Message message, ResultProcessor`1 processor, ServerEndPoint server)
   at StackExchange.Redis.RedisBase.ExecuteSync[T](Message message, ResultProcessor`1 processor, ServerEndPoint server)
   at MyASPMVCRevamp.Service.RedisCacheHelper.Set(String cacheKey, Object ObjValue, Int32 cacheMinutes) in D:\Work\\MyASPMVCFronts\MyASPMVCRevamp.sln\Master\MyASPMVCRevamp.Service\RedisService.cs:line 40
   at MyASPMVCRevamp.Service.GeneralServices.GetStaticPageByID(Int32 ID) in D:\Work\\MyASPMVCFronts\MyASPMVCRevamp.sln\Master\MyASPMVCRevamp.Service\GeneralServices.cs:line 353
   at lambda_method(Closure , ControllerBase , Object[] )
   at System.Web.Mvc.ControllerActionInvoker.InvokeActionMethod(ControllerContext controllerContext, ActionDescriptor actionDescriptor, IDictionary`2 parameters)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.<>c.<BeginInvokeSynchronousActionMethod>b__9_0(IAsyncResult asyncResult, ActionInvocation innerInvokeState)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncResult`2.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.EndInvokeActionMethod(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.AsyncInvocationWithFilters.<InvokeActionMethodFilterAsynchronouslyRecursive>b__11_0()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.AsyncInvocationWithFilters.<>c__DisplayClass11_1.<InvokeActionMethodFilterAsynchronouslyRecursive>b__2()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.AsyncInvocationWithFilters.<>c__DisplayClass11_1.<InvokeActionMethodFilterAsynchronouslyRecursive>b__2()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.EndInvokeActionMethodWithFilters(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.<>c__DisplayClass3_6.<BeginInvokeAction>b__3()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.<>c__DisplayClass3_1.<BeginInvokeAction>b__5(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.EndInvokeAction(IAsyncResult asyncResult)
   at System.Web.Mvc.Controller.<>c.<BeginExecuteCore>b__152_1(IAsyncResult asyncResult, ExecuteCoreState innerState)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncVoid`1.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.Controller.EndExecuteCore(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncVoid`1.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.Controller.EndExecute(IAsyncResult asyncResult)
   at System.Web.Mvc.MvcHandler.<>c.<BeginProcessRequest>b__20_1(IAsyncResult asyncResult, ProcessRequestState innerState)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncVoid`1.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.MvcHandler.EndProcessRequest(IAsyncResult asyncResult)
   at System.Web.Mvc.HttpHandlerUtil.ServerExecuteHttpHandlerWrapper.<>c__DisplayClass5_0.<Wrap>b__0()
   at System.Web.Mvc.HttpHandlerUtil.ServerExecuteHttpHandlerWrapper.Wrap[TResult](Func`1 func)
   at System.Web.HttpServerUtility.ExecuteInternal(IHttpHandler handler, TextWriter writer, Boolean preserveForm, Boolean setPreviousPage, VirtualPath path, VirtualPath filePath, String physPath, Exception error, String queryStringOverride)

 
 
Request information: 
    Request URL: https://www.MyASPMVC.com:443/en-us/elogin
    Request path: /en-us/elogin 
    User host address: 
    User:  
    Is authenticated: False 
    Authentication Type:  
    Thread account name: EC2AMAZ-VG4448R\MyASPMVC_web 
 
Thread information: 
    Thread ID: 640 
    Thread account name: EC2AMAZ-VG4448R\MyASPMVC_web 
    Is impersonating: False 
    Stack trace:    at StackExchange.Redis.ConnectionMultiplexer.ExecuteSyncImpl[T](Message message, ResultProcessor`1 processor, ServerEndPoint server)
   at StackExchange.Redis.RedisBase.ExecuteSync[T](Message message, ResultProcessor`1 processor, ServerEndPoint server)
   at MyASPMVCRevamp.Service.RedisCacheHelper.Set(String cacheKey, Object ObjValue, Int32 cacheMinutes) in D:\Work\\MyASPMVCFronts\MyASPMVCRevamp.sln\Master\MyASPMVCRevamp.Service\RedisService.cs:line 40
   at MyASPMVCRevamp.Service.GeneralServices.GetStaticPageByID(Int32 ID) in D:\Work\\MyASPMVCFronts\MyASPMVCRevamp.sln\Master\MyASPMVCRevamp.Service\GeneralServices.cs:line 353
   at lambda_method(Closure , ControllerBase , Object[] )
   at System.Web.Mvc.ControllerActionInvoker.InvokeActionMethod(ControllerContext controllerContext, ActionDescriptor actionDescriptor, IDictionary`2 parameters)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.<>c.<BeginInvokeSynchronousActionMethod>b__9_0(IAsyncResult asyncResult, ActionInvocation innerInvokeState)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncResult`2.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.EndInvokeActionMethod(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.AsyncInvocationWithFilters.<InvokeActionMethodFilterAsynchronouslyRecursive>b__11_0()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.AsyncInvocationWithFilters.<>c__DisplayClass11_1.<InvokeActionMethodFilterAsynchronouslyRecursive>b__2()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.AsyncInvocationWithFilters.<>c__DisplayClass11_1.<InvokeActionMethodFilterAsynchronouslyRecursive>b__2()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.EndInvokeActionMethodWithFilters(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.<>c__DisplayClass3_6.<BeginInvokeAction>b__3()
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.<>c__DisplayClass3_1.<BeginInvokeAction>b__5(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncControllerActionInvoker.EndInvokeAction(IAsyncResult asyncResult)
   at System.Web.Mvc.Controller.<>c.<BeginExecuteCore>b__152_1(IAsyncResult asyncResult, ExecuteCoreState innerState)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncVoid`1.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.Controller.EndExecuteCore(IAsyncResult asyncResult)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncVoid`1.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.Controller.EndExecute(IAsyncResult asyncResult)
   at System.Web.Mvc.MvcHandler.<>c.<BeginProcessRequest>b__20_1(IAsyncResult asyncResult, ProcessRequestState innerState)
   at System.Web.Mvc.Async.AsyncResultWrapper.WrappedAsyncVoid`1.CallEndDelegate(IAsyncResult asyncResult)
   at System.Web.Mvc.MvcHandler.EndProcessRequest(IAsyncResult asyncResult)
   at System.Web.Mvc.HttpHandlerUtil.ServerExecuteHttpHandlerWrapper.<>c__DisplayClass5_0.<Wrap>b__0()
   at System.Web.Mvc.HttpHandlerUtil.ServerExecuteHttpHandlerWrapper.Wrap[TResult](Func`1 func)
   at System.Web.HttpServerUtility.ExecuteInternal(IHttpHandler handler, TextWriter writer, Boolean preserveForm, Boolean setPreviousPage, VirtualPath path, VirtualPath filePath, String physPath, Exception error, String queryStringOverride)
 
 
Custom event details: 

and searched alot for a solution and tried alot but cannot find anything would help in that case

Answer 1

As I can see from your exception message, your minimum worker process count is too low for the traffic you have

WORKER: (Busy=647,Free=32120,Min=16,Max=32767)

You had 647 busy worker threads when this exception happened, while you had 2 worker threads for start.

You could use ThreadPool.SetMinThreads(Int32, Int32) method at the beginning of your application to set minimum thread count. I suggest you to start with ThreadPool.SetMinThreads(1000, 1000) and tweak it as you test it.

More Info https://learn.microsoft.com/en-us/do.net/api/system.threading.threadpool.setminthreads?view.net-7.0