Configure min and max IO completion threads (#904)

* Configure min and max IO completion threads separately from min and max threads (in the ThreadPool). This is needed as some scenarios may limit number of thread pool threads but require a larger number of IO completion threads.

* nit

libs/host/Configuration/Options.cs

@@ -332,13 +332,21 @@ internal sealed class Options
         public string FileLogger { get; set; }
 
         [IntRangeValidation(0, int.MaxValue)]
-        [Option("minthreads", Required = false, HelpText = "Minimum worker and completion threads in thread pool, 0 uses the system default.")]
+        [Option("minthreads", Required = false, HelpText = "Minimum worker threads in thread pool, 0 uses the system default.")]
         public int ThreadPoolMinThreads { get; set; }
 
         [IntRangeValidation(0, int.MaxValue)]
-        [Option("maxthreads", Required = false, HelpText = "Maximum worker and completion threads in thread pool, 0 uses the system default.")]
+        [Option("maxthreads", Required = false, HelpText = "Maximum worker threads in thread pool, 0 uses the system default.")]
         public int ThreadPoolMaxThreads { get; set; }
 
+        [IntRangeValidation(0, int.MaxValue)]
+        [Option("miniothreads", Required = false, HelpText = "Minimum IO completion threads in thread pool, 0 uses the system default.")]
+        public int ThreadPoolMinIOCompletionThreads { get; set; }
+
+        [IntRangeValidation(0, int.MaxValue)]
+        [Option("maxiothreads", Required = false, HelpText = "Maximum IO completion threads in thread pool, 0 uses the system default.")]
+        public int ThreadPoolMaxIOCompletionThreads { get; set; }
+
         [IntRangeValidation(-1, int.MaxValue)]
         [Option("network-connection-limit", Required = false, Default = -1, HelpText = "Maximum number of simultaneously active network connections.")]
         public int NetworkConnectionLimit { get; set; }
@@ -680,6 +688,8 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 QuietMode = QuietMode.GetValueOrDefault(),
                 ThreadPoolMinThreads = ThreadPoolMinThreads,
                 ThreadPoolMaxThreads = ThreadPoolMaxThreads,
+                ThreadPoolMinIOCompletionThreads = ThreadPoolMinIOCompletionThreads,
+                ThreadPoolMaxIOCompletionThreads = ThreadPoolMaxIOCompletionThreads,
                 NetworkConnectionLimit = NetworkConnectionLimit,
                 DeviceFactoryCreator = useAzureStorage
                     ? () => new AzureStorageNamedDeviceFactory(AzureStorageConnectionString, logger)

libs/host/GarnetServer.cs

@@ -188,14 +188,41 @@ private void InitializeServer()
 
             var customCommandManager = new CustomCommandManager();
 
-            var setMax = opts.ThreadPoolMaxThreads <= 0 || ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads);
+            ThreadPool.GetMinThreads(out var minThreads, out var minCPThreads);
+            ThreadPool.GetMaxThreads(out var maxThreads, out var maxCPThreads);
 
-            if (opts.ThreadPoolMinThreads > 0 && !ThreadPool.SetMinThreads(opts.ThreadPoolMinThreads, opts.ThreadPoolMinThreads))
-                throw new Exception($"Unable to call ThreadPool.SetMinThreads with {opts.ThreadPoolMinThreads}");
+            bool minChanged = false, maxChanged = false;
+            if (opts.ThreadPoolMinThreads > 0)
+            {
+                minThreads = opts.ThreadPoolMinThreads;
+                minChanged = true;
+            }
+            if (opts.ThreadPoolMinIOCompletionThreads > 0)
+            {
+                minCPThreads = opts.ThreadPoolMinIOCompletionThreads;
+                minChanged = true;
+            }
+            if (opts.ThreadPoolMaxThreads > 0)
+            {
+                maxThreads = opts.ThreadPoolMaxThreads;
+                maxChanged = true;
+            }
+            if (opts.ThreadPoolMaxIOCompletionThreads > 0)
+            {
+                maxCPThreads = opts.ThreadPoolMaxIOCompletionThreads;
+                maxChanged = true;
+            }
+
+            // First try to set the max threads
+            var setMax = !maxChanged || ThreadPool.SetMaxThreads(maxThreads, maxCPThreads);
+
+            // Set the min threads
+            if (minChanged && !ThreadPool.SetMinThreads(minThreads, minCPThreads))
+                throw new Exception($"Unable to call ThreadPool.SetMinThreads with {minThreads}, {minCPThreads}");
 
-            // Retry to set max threads if it wasn't set in the previous step
-            if (!setMax && !ThreadPool.SetMaxThreads(opts.ThreadPoolMaxThreads, opts.ThreadPoolMaxThreads))
-                throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {opts.ThreadPoolMaxThreads}");
+            // Retry to set max threads if it wasn't set in the earlier step
+            if (!setMax && !ThreadPool.SetMaxThreads(maxThreads, maxCPThreads))
+                throw new Exception($"Unable to call ThreadPool.SetMaxThreads with {maxThreads}, {maxCPThreads}");
 
             CreateMainStore(clusterFactory, out var checkpointDir);
             CreateObjectStore(clusterFactory, customCommandManager, checkpointDir, out var objectStoreSizeTracker);

libs/host/defaults.conf

@@ -245,12 +245,18 @@
 	/* Enable file logger and write to the specified path. */
 	"FileLogger" : null,
 
-	/* Minimum worker and completion threads in thread pool, 0 uses the system default. */
+	/* Minimum worker threads in thread pool, 0 uses the system default. */
 	"ThreadPoolMinThreads" : 0,
 
-	/* Maximum worker and completion threads in thread pool, 0 uses the system default. */
+	/* Maximum worker threads in thread pool, 0 uses the system default. */
 	"ThreadPoolMaxThreads" : 0,
 
+	/* Minimum IO completion threads in thread pool, 0 uses the system default. */
+	"ThreadPoolMinIOCompletionThreads" : 0,
+
+	/* Maximum IO completion threads in thread pool, 0 uses the system default. */
+	"ThreadPoolMaxIOCompletionThreads" : 0,
+
 	/* Maximum number of simultaneously active network connections. */
 	"NetworkConnectionLimit" : -1,
 

libs/server/Servers/GarnetServerOptions.cs

@@ -237,15 +237,25 @@ public class GarnetServerOptions : ServerOptions
         public bool UseFoldOverCheckpoints = false;
 
         /// <summary>
-        /// Minimum worker and completion port threads in thread pool (0 for default)
+        /// Minimum worker threads in thread pool (0 for default)
         /// </summary>
         public int ThreadPoolMinThreads = 0;
 
         /// <summary>
-        /// Maximum worker and completion port threads in thread pool (0 for default)
+        /// Maximum worker threads in thread pool (0 for default)
         /// </summary>
         public int ThreadPoolMaxThreads = 0;
 
+        /// <summary>
+        /// Minimum IO completion threads in thread pool (0 for default)
+        /// </summary>
+        public int ThreadPoolMinIOCompletionThreads = 0;
+
+        /// <summary>
+        /// Maximum IO completion threads in thread pool (0 for default)
+        /// </summary>
+        public int ThreadPoolMaxIOCompletionThreads = 0;
+
         /// <summary>
         /// Maximum client connection limit
         /// </summary>