Gettting error Timeout performing ZADD (10000ms), next: HSCAN in .net core 3.0 application with redis version 4

Question

I am running my .net core 3.0 application in ubuntu 18.04 machine in same network where redis present I am getting error

Timeout performing ZADD (10000ms), next: HSCAN InProccessingMsisdnTransaction_34234234, inst: 1, qu: 0, qs: 81, aw: False, rs: ReadAsync, ws: Idle, in: 34117, in-pipe: 0, out-pipe: 0, serverEndpoint: 10.10.10.10:6379, mgr: 10 of 10 available, clientName: SafeRedisConnection, IOCP: (Busy=0,Free=1000,Min=10,Max=1000), WORKER: (Busy=32,Free=32735,Min=2,Max=32767), v: 2.0.601.3402 (Please take a look at this article for some common client-side issues that can cause timeouts: https://stackexchange.github.io/StackExchange.Redis/Timeouts )

My implementation for redis interaction is as following

public static class RedisManager
    {

        private static readonly bool LogRedisRelatedActivities;
           private static readonly Lazy<ConfigurationOptions> ConfigOptions = new Lazy<ConfigurationOptions>(() =>
        {
            ConfigurationOptions configOptions = null;

            try
            {
                var redisInfo = ConfigurationHandler.GetSection<RedisElement>("RedisSection");

                if (redisInfo != null )
                {
                    if (redisInfo.IsActive)
                    {
                        redisInfo.RedisServerNameOrIp = ConfigurationHandler.GetSection<string>(StringConstants.EnvConfig.RedisRedisServerNameOrIp, Configurationtype.RedisSection);
                        redisInfo.RedisServerPort = ConfigurationHandler.GetSection<string>(StringConstants.EnvConfig.RedisRedisServerPort, Configurationtype.RedisSection);
                        redisInfo.RedisDefaultDatabase = ConfigurationHandler.GetSection<int>(StringConstants.EnvConfig.RedisDefaultDatabase, Configurationtype.RedisSection);


                        configOptions = new ConfigurationOptions();
                        configOptions.EndPoints.Add(redisInfo.RedisServerNameOrIp + ":" + redisInfo.RedisServerPort);
                        configOptions.ClientName = "SafeRedisConnection";

                        configOptions.ConnectTimeout = redisInfo.ConnectTimeout * 1000;
                        configOptions.SyncTimeout = redisInfo.SyncTimeout * 1000;
                        configOptions.AbortOnConnectFail = false;
                        configOptions.KeepAlive = redisInfo.KeepAliveDuration;
                        configOptions.DefaultDatabase = redisInfo.RedisDefaultDatabase;

                    }
                    else
                    {
                        Logger.Error("RedisSection is in-active");
                    }
                }
                else
                {
                    Logger.Error("RedisSection not found");
                }
            }
            catch (Exception ex)
            {
                Logger.Fatal(ex, ex);
            }
            return configOptions;
        });
        private static readonly Lazy<ConnectionMultiplexer> Conn = new Lazy<ConnectionMultiplexer>(
            () =>
            {
                try
                {
                    if (ConfigOptions != null && ConfigOptions.Value != null)
                    {
                        return ConnectionMultiplexer.Connect(ConfigOptions.Value);
                    }
                    return null;
                }
                catch (Exception ex)
                {
                    Logger.Fatal(ex.Message, ex);
                    return null;
                }
            });

        private static ConnectionMultiplexer Muxer => Conn.Value;

        static RedisManager()
        {
            try
            {
                LogRedisRelatedActivities = ConfigurationHandler.GetSection<bool>(StringConstants.AppSettingsKeys.LogRedisRelatedActivities);

                if (Muxer != null && Muxer.IsConnected)
                {
                    Logger.Info("Redis Connected ");
                }
                else
                {
                    Logger.Info("Redis Not Connected ");
                }

            }
            catch (Exception ex)
            {
                Logger.Fatal(ex.Message, ex);
            }
        }

        public static string GetStringItem(string key)
        {
            string val = null;
            try
            {
                IDatabase getDatabase;
                if (Muxer != null && Muxer.IsConnected && (getDatabase = Muxer.GetDatabase()) != null)
                {
                    val = getDatabase.StringGet(key);

                }
            }
            catch (Exception ex)
            {
                Logger.Fatal(ex.Message, ex);
            }
            return val;
        }       

        public static bool AddZList(string key, object value)
        {
            var isAdded = false;
            try
            {
                IDatabase getDatabase;
                if (Muxer != null && Muxer.IsConnected && (getDatabase = Muxer.GetDatabase()) != null)
                {
                    isAdded = getDatabase.SortedSetAdd(key, JsonSerializer.Serialize(value), Utility.GetCurrentSystemTime().Ticks);

                }
            }
            catch (Exception ex)
            {
                Logger.Fatal(ex.Message, ex);
            }
            return isAdded;
        }      
    }

When running same application from windows server i am not getting any such error

Answer 1

Problem was related to some threadpool implemenation in stackexchange.redis. Workaround is to initialize high number of minimum thread in our application. done it via some configuration in .csproj file