From 224dde7ef119959ffc2d18420d7b9e76121faa19 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 19 Aug 2019 18:12:39 -0700 Subject: [PATCH] Allow user disable multiple threading (#1647) --- .../test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs | 3 +-- include/onnxruntime/core/session/onnxruntime_c_api.h | 7 ++++++- onnxruntime/core/session/abi_session_options.cc | 2 -- onnxruntime/core/session/inference_session.cc | 3 +-- onnxruntime/core/session/inference_session.h | 2 +- onnxruntime/test/perftest/command_args_parser.cc | 7 ++++--- onnxruntime/test/perftest/ort_test_session.cc | 4 +--- onnxruntime/test/perftest/test_configuration.h | 2 +- 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 15986bbb60..fd8ddc45da 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -63,8 +63,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests opt.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_EXTENDED; Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, opt.GraphOptimizationLevel); - - Assert.Throws(() => { opt.ThreadPoolSize = -2; }); + Assert.Throws(() => { opt.GraphOptimizationLevel = (GraphOptimizationLevel)10; }); } } diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 8566125ac1..6025a8ac80 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -247,7 +247,12 @@ typedef enum GraphOptimizationLevel { ORT_API_STATUS(OrtSetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, GraphOptimizationLevel graph_optimization_level); -// How many threads in the session thread pool. +/** + * How many threads in the session thread pool. + * Set it to 0 to make onnxruntime run as single threaded. + * \param session_thread_pool_size <0, let the runtime choose a default. =0, Don't create extra threads. + * >0, create a thread pool with size of this value. + */ ORT_API_STATUS(OrtSetSessionThreadPoolSize, _Inout_ OrtSessionOptions* options, int session_thread_pool_size); /** diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index 647e5c3722..214c6b6818 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -129,8 +129,6 @@ ORT_API_STATUS_IMPL(OrtSetSessionGraphOptimizationLevel, _In_ OrtSessionOptions* ///How many threads in the session thread pool. ORT_API_STATUS_IMPL(OrtSetSessionThreadPoolSize, _In_ OrtSessionOptions* options, int session_thread_pool_size) { - if (session_thread_pool_size <= 0) - return OrtCreateStatus(ORT_INVALID_ARGUMENT, "session_thread_pool_size is not valid"); options->value.session_thread_pool_size = session_thread_pool_size; return nullptr; } diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 6dc370b797..4d8fa70784 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -92,8 +92,7 @@ inline std::basic_string GetCurrentTimeString() { } concurrency::ThreadPool* CreateThreadPool(int size) { - if (size <= 0) - size = std::thread::hardware_concurrency() / 2; + if (size < 0) size = std::thread::hardware_concurrency() / 2; return size > 0 ? new concurrency::ThreadPool("SESSION", size) : nullptr; } diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index 7fcbe0ad87..632c92105b 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -88,7 +88,7 @@ struct SessionOptions { TransformerLevel graph_optimization_level = TransformerLevel::Level1; // How many threads in the session thread pool. - int session_thread_pool_size = 0; + int session_thread_pool_size = -1; }; /** diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index ee54ee4b55..7e70a956f6 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -32,7 +32,8 @@ namespace perftest { "\t-M: Disable memory pattern.\n" "\t-A: Disable memory arena\n" "\t-c [parallel runs]: Specifies the (max) number of runs to invoke simultaneously. Default:1.\n" - "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', 'ngraph' or 'openvino'. " + "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', " + "'ngraph' or 'openvino'. " "Default:'cpu'.\n" "\t-b [tf|ort]: backend to use. Default:ort\n" "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n" @@ -40,7 +41,7 @@ namespace perftest { "\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n" "\t-s: Show statistics result, like P75, P90.\n" "\t-v: Show verbose information.\n" - "\t-x [thread_size]: Session thread pool size.\n" + "\t-x [thread_size]: Session thread pool size, must >=0.\n" "\t-P: Use parallel executor instead of sequential executor.\n" "\t-o [optimization level]: Please see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. \n" "\t-h: help\n"); @@ -114,7 +115,7 @@ namespace perftest { break; case 'x': test_config.run_config.session_thread_pool_size = static_cast(OrtStrtol(optarg, nullptr)); - if (test_config.run_config.session_thread_pool_size <= 0) { + if (test_config.run_config.session_thread_pool_size < 0) { return false; } break; diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index d74fb1ba9a..f798cfaff7 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -91,9 +91,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device else session_options.DisableSequentialExecution(); fprintf(stdout, "Setting thread pool size to %d\n", performance_test_config.run_config.session_thread_pool_size); - // Don't set the thread pool size unless it has been changed from our zero default value (as zero will fail) - if (performance_test_config.run_config.session_thread_pool_size != 0) - session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size); + session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size); // Set optimization level. session_options.SetGraphOptimizationLevel(performance_test_config.run_config.optimization_level); if (!performance_test_config.run_config.profile_file.empty()) diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index 95f793bc94..1ab5010948 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -44,7 +44,7 @@ struct RunConfig { bool enable_memory_pattern{true}; bool enable_cpu_mem_arena{true}; bool enable_sequential_execution{true}; - int session_thread_pool_size{0}; + int session_thread_pool_size{-1}; GraphOptimizationLevel optimization_level{ORT_ENABLE_EXTENDED}; };