Allow user disable multiple threading (#1647)

csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs

@@ -63,8 +63,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
 
                 opt.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_EXTENDED;
                 Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, opt.GraphOptimizationLevel);
-
-                Assert.Throws<OnnxRuntimeException>(() => { opt.ThreadPoolSize = -2; });
+                
                 Assert.Throws<OnnxRuntimeException>(() => { opt.GraphOptimizationLevel = (GraphOptimizationLevel)10; });
             }
         }

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -247,7 +247,12 @@ typedef enum GraphOptimizationLevel {
 ORT_API_STATUS(OrtSetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options,
                GraphOptimizationLevel graph_optimization_level);
 
-// How many threads in the session thread pool.
+/**
+ * How many threads in the session thread pool.
+ * Set it to 0 to make onnxruntime run as single threaded.
+ * \param session_thread_pool_size <0, let the runtime choose a default. =0, Don't create extra threads. 
+ *                                 >0, create a thread pool with size of this value.
+ */
 ORT_API_STATUS(OrtSetSessionThreadPoolSize, _Inout_ OrtSessionOptions* options, int session_thread_pool_size);
 
 /**

onnxruntime/core/session/abi_session_options.cc

@@ -129,8 +129,6 @@ ORT_API_STATUS_IMPL(OrtSetSessionGraphOptimizationLevel, _In_ OrtSessionOptions*
 
 ///How many threads in the session thread pool.
 ORT_API_STATUS_IMPL(OrtSetSessionThreadPoolSize, _In_ OrtSessionOptions* options, int session_thread_pool_size) {
-  if (session_thread_pool_size <= 0)
-    return OrtCreateStatus(ORT_INVALID_ARGUMENT, "session_thread_pool_size is not valid");
   options->value.session_thread_pool_size = session_thread_pool_size;
   return nullptr;
 }

onnxruntime/core/session/inference_session.cc

@@ -92,8 +92,7 @@ inline std::basic_string<T> GetCurrentTimeString() {
 }
 
 concurrency::ThreadPool* CreateThreadPool(int size) {
-  if (size <= 0)
-    size = std::thread::hardware_concurrency() / 2;
+  if (size < 0) size = std::thread::hardware_concurrency() / 2;
   return size > 0 ? new concurrency::ThreadPool("SESSION", size) : nullptr;
 }
 

onnxruntime/core/session/inference_session.h

@@ -88,7 +88,7 @@ struct SessionOptions {
   TransformerLevel graph_optimization_level = TransformerLevel::Level1;
 
   // How many threads in the session thread pool.
-  int session_thread_pool_size = 0;
+  int session_thread_pool_size = -1;
 };
 
 /**

onnxruntime/test/perftest/command_args_parser.cc

@@ -32,7 +32,8 @@ namespace perftest {
       "\t-M: Disable memory pattern.\n"
       "\t-A: Disable memory arena\n"
       "\t-c [parallel runs]: Specifies the (max) number of runs to invoke simultaneously. Default:1.\n"
-      "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', 'ngraph' or 'openvino'. "
+      "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', "
+      "'ngraph' or 'openvino'. "
       "Default:'cpu'.\n"
       "\t-b [tf|ort]: backend to use. Default:ort\n"
       "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n"
@@ -40,7 +41,7 @@ namespace perftest {
       "\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n"
       "\t-s: Show statistics result, like P75, P90.\n"
       "\t-v: Show verbose information.\n"
-      "\t-x [thread_size]: Session thread pool size.\n"
+      "\t-x [thread_size]: Session thread pool size, must >=0.\n"
       "\t-P: Use parallel executor instead of sequential executor.\n"
       "\t-o [optimization level]: Please see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. \n"
       "\t-h: help\n");
@@ -114,7 +115,7 @@ namespace perftest {
         break;
       case 'x':
         test_config.run_config.session_thread_pool_size = static_cast<int>(OrtStrtol<PATH_CHAR_TYPE>(optarg, nullptr));
-        if (test_config.run_config.session_thread_pool_size <= 0) {
+        if (test_config.run_config.session_thread_pool_size < 0) {
           return false;
         }
         break;

onnxruntime/test/perftest/ort_test_session.cc

@@ -91,9 +91,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
   else
     session_options.DisableSequentialExecution();
   fprintf(stdout, "Setting thread pool size to %d\n", performance_test_config.run_config.session_thread_pool_size);
-  // Don't set the thread pool size unless it has been changed from our zero default value (as zero will fail)
-  if (performance_test_config.run_config.session_thread_pool_size != 0)
-    session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size);
+  session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size);
   // Set optimization level.
   session_options.SetGraphOptimizationLevel(performance_test_config.run_config.optimization_level);
   if (!performance_test_config.run_config.profile_file.empty())

onnxruntime/test/perftest/test_configuration.h

@@ -44,7 +44,7 @@ struct RunConfig {
   bool enable_memory_pattern{true};
   bool enable_cpu_mem_arena{true};
   bool enable_sequential_execution{true};
-  int session_thread_pool_size{0};
+  int session_thread_pool_size{-1};
   GraphOptimizationLevel optimization_level{ORT_ENABLE_EXTENDED};
 };